diff --git a/src/controllers/SalaryPeriodController.ts b/src/controllers/SalaryPeriodController.ts index d7525ac..ec67bd8 100644 --- a/src/controllers/SalaryPeriodController.ts +++ b/src/controllers/SalaryPeriodController.ts @@ -2563,11 +2563,16 @@ export class SalaryPeriodController extends Controller { errorMessage, ); - // Check if it's a connection error + // Check if it's a connection or SSL error if ( errorMessage.includes("ECONNRESET") || errorMessage.includes("Connection lost") || - errorMessage.includes("ETIMEDOUT") + errorMessage.includes("ETIMEDOUT") || + errorMessage.includes("SSL routines") || + errorMessage.includes("decryption failed") || + errorMessage.includes("bad record mac") || + errorMessage.includes("EPIPE") || + errorMessage.includes("socket hang up") ) { retryCount++; if (retryCount < maxRetries) { @@ -2599,6 +2604,22 @@ export class SalaryPeriodController extends Controller { salaryPeriodId: string, request: RequestWithUser, ) { + // ปรับ connection timeout เพื่อรองรับการประมวลผลข้อมูลขนาดใหญ่ + const connection = AppDataSource; + if (connection.isInitialized) { + try { + // ปรับ query timeout สำหรับ long-running operations + await connection.query("SET SESSION wait_timeout = 3600"); // 1 hour + await connection.query("SET SESSION interactive_timeout = 3600"); + await connection.query("SET SESSION max_execution_time = 0"); // No limit + await connection.query("SET SESSION net_read_timeout = 600"); // 10 minutes + await connection.query("SET SESSION net_write_timeout = 600"); // 10 minutes + console.log("✅ ปรับ database session timeout เรียบร้อย"); + } catch (sessionErr) { + console.warn("⚠️ ไม่สามารถปรับ session timeout ได้:", sessionErr); + } + } + const salaryPeriod = await this.salaryPeriodRepository.findOne({ where: { id: salaryPeriodId }, }); @@ -2606,60 +2627,221 @@ export class SalaryPeriodController extends Controller { throw new HttpError(HttpStatusCode.NOT_FOUND, "ไม่พบรอบการสร้างเงินเดือน"); } - // Parallel loading of existing data for removal - console.time("⏱ Cleanup: Load existing data"); - const [salaryOrg, salaryOrgEmployee] = await Promise.all([ - this.salaryOrgRepository.find({ - where: { salaryPeriodId: salaryPeriod.id, snapshot: snapshot }, - }), - this.salaryOrgEmployeeRepository.find({ - where: { salaryPeriodId: salaryPeriod.id, snapshot: snapshot }, - }), - ]); + // Fast bulk delete approach - ไม่ต้อง load ข้อมูลมาก่อนลบ + console.time("⏱ Cleanup: Fast bulk delete"); - // Parallel loading of profiles for removal - const [salaryProfile, salaryProfileEmployee] = await Promise.all([ - salaryOrg.length > 0 - ? this.salaryProfileRepository.find({ - where: { salaryOrgId: In(salaryOrg.map((x) => x.id)) }, - }) - : Promise.resolve([]), - salaryOrgEmployee.length > 0 - ? this.salaryProfileEmployeeRepository.find({ - where: { salaryOrgId: In(salaryOrgEmployee.map((x) => x.id)) }, - }) - : Promise.resolve([]), - ]); - console.timeEnd("⏱ Cleanup: Load existing data"); + // สำหรับข้อมูลจำนวนมาก ใช้ TRUNCATE หรือ bulk delete + // Option 1: ถ้าต้องการลบข้อมูลทั้งหมดของ snapshot + if (snapshot === "SNAP1" || snapshot === "SNAP2") { + console.log(`🚀 ใช้ Super Fast Delete สำหรับ ${snapshot}`); - console.log( - `🧹 ลบข้อมูลเก่า: SalaryProfile=${salaryProfile.length}, SalaryProfileEmployee=${salaryProfileEmployee.length}, SalaryOrg=${salaryOrg.length}, SalaryOrgEmployee=${salaryOrgEmployee.length}`, - ); + // Super fast approach: ลบแบบ sequential เพื่อเคารพ foreign key constraints + // แต่ใช้ bulk delete แทนการ load + remove - // Sequential removal to respect foreign key constraints - // First, remove all profiles (child records) - console.time("⏱ Cleanup: Remove child records"); - await Promise.all([ - salaryProfile.length > 0 - ? this.salaryProfileRepository.remove(salaryProfile, { data: request }) - : Promise.resolve(), - salaryProfileEmployee.length > 0 - ? this.salaryProfileEmployeeRepository.remove(salaryProfileEmployee, { data: request }) - : Promise.resolve(), - ]); - console.timeEnd("⏱ Cleanup: Remove child records"); + // For extremely large datasets (100K+ records), use batch delete to prevent timeouts + const isLargeDataset = await AppDataSource.query( + ` + SELECT COUNT(*) as count FROM salaryOrg WHERE salaryPeriodId = ? AND snapshot = ? + `, + [salaryPeriod.id, snapshot], + ).then((result) => result[0]?.count > 50000); - // Then, remove the org records (parent records) - console.time("⏱ Cleanup: Remove parent records"); - await Promise.all([ - salaryOrg.length > 0 - ? this.salaryOrgRepository.remove(salaryOrg, { data: request }) - : Promise.resolve(), - salaryOrgEmployee.length > 0 - ? this.salaryOrgEmployeeRepository.remove(salaryOrgEmployee, { data: request }) - : Promise.resolve(), - ]); - console.timeEnd("⏱ Cleanup: Remove parent records"); + if (isLargeDataset) { + console.log(`🔥 ข้อมูลขนาดใหญ่มาก - ใช้ Batch Delete`); + + // Batch delete for large datasets to prevent timeout + let deletedProfiles = 0, + deletedProfileEmployees = 0; + + // Delete profiles in batches + while (true) { + const result = await AppDataSource.query( + ` + DELETE sp FROM salaryProfile sp + INNER JOIN salaryOrg so ON sp.salaryOrgId = so.id + WHERE so.salaryPeriodId = ? AND so.snapshot = ? + LIMIT 10000 + `, + [salaryPeriod.id, snapshot], + ); + + deletedProfiles += result.affectedRows; + if (result.affectedRows === 0) break; + + // Small delay to prevent overwhelming the database + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + // Delete profile employees in batches + while (true) { + const result = await AppDataSource.query( + ` + DELETE spe FROM salaryProfileEmployee spe + INNER JOIN salaryOrgEmployee soe ON spe.salaryOrgId = soe.id + WHERE soe.salaryPeriodId = ? AND soe.snapshot = ? + LIMIT 10000 + `, + [salaryPeriod.id, snapshot], + ); + + deletedProfileEmployees += result.affectedRows; + if (result.affectedRows === 0) break; + + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + // Delete orgs and org employees + const [deletedOrg, deletedOrgEmployee] = await Promise.all([ + AppDataSource.query(`DELETE FROM salaryOrg WHERE salaryPeriodId = ? AND snapshot = ?`, [ + salaryPeriod.id, + snapshot, + ]), + AppDataSource.query( + `DELETE FROM salaryOrgEmployee WHERE salaryPeriodId = ? AND snapshot = ?`, + [salaryPeriod.id, snapshot], + ), + ]); + + console.log( + `✅ Batch Delete เสร็จสิ้น - Profiles: ${deletedProfiles}, ProfileEmployees: ${deletedProfileEmployees}, Orgs: ${deletedOrg.affectedRows}, OrgEmployees: ${deletedOrgEmployee.affectedRows}`, + ); + } else { + // Standard fast delete for smaller datasets + // Step 1: ลบ SalaryProfile ก่อน + const deletedProfiles = await AppDataSource.query( + ` + DELETE sp FROM salaryProfile sp + INNER JOIN salaryOrg so ON sp.salaryOrgId = so.id + WHERE so.salaryPeriodId = ? AND so.snapshot = ? + `, + [salaryPeriod.id, snapshot], + ); + + // Step 2: ลบ SalaryProfileEmployee + const deletedProfileEmployees = await AppDataSource.query( + ` + DELETE spe FROM salaryProfileEmployee spe + INNER JOIN salaryOrgEmployee soe ON spe.salaryOrgId = soe.id + WHERE soe.salaryPeriodId = ? AND soe.snapshot = ? + `, + [salaryPeriod.id, snapshot], + ); + + // Step 3: ลบ SalaryOrg และ SalaryOrgEmployee + const [deletedOrg, deletedOrgEmployee] = await Promise.all([ + AppDataSource.query(`DELETE FROM salaryOrg WHERE salaryPeriodId = ? AND snapshot = ?`, [ + salaryPeriod.id, + snapshot, + ]), + AppDataSource.query( + `DELETE FROM salaryOrgEmployee WHERE salaryPeriodId = ? AND snapshot = ?`, + [salaryPeriod.id, snapshot], + ), + ]); + + console.log( + `✅ Super Fast Delete เสร็จสิ้น - Profiles: ${deletedProfiles.affectedRows}, ProfileEmployees: ${deletedProfileEmployees.affectedRows}, Orgs: ${deletedOrg.affectedRows}, OrgEmployees: ${deletedOrgEmployee.affectedRows}`, + ); + } + } else { + // Fallback: Standard bulk delete approach สำหรับ snapshots อื่นๆ + console.log(`📊 ใช้ Standard Bulk Delete สำหรับ ${snapshot}`); + + // Get counts for logging (optional, can be removed for even better performance) + const [salaryOrgCount, salaryOrgEmployeeCount] = await Promise.all([ + this.salaryOrgRepository.count({ + where: { salaryPeriodId: salaryPeriod.id, snapshot: snapshot }, + }), + this.salaryOrgEmployeeRepository.count({ + where: { salaryPeriodId: salaryPeriod.id, snapshot: snapshot }, + }), + ]); + + if (salaryOrgCount > 0 || salaryOrgEmployeeCount > 0) { + // Get profile counts using subqueries + const [salaryProfileCount, salaryProfileEmployeeCount] = await Promise.all([ + salaryOrgCount > 0 + ? AppDataSource.query( + ` + SELECT COUNT(*) as count FROM salaryProfile + WHERE salaryOrgId IN ( + SELECT id FROM salaryOrg + WHERE salaryPeriodId = ? AND snapshot = ? + ) + `, + [salaryPeriod.id, snapshot], + ).then((result) => result[0]?.count || 0) + : Promise.resolve(0), + salaryOrgEmployeeCount > 0 + ? AppDataSource.query( + ` + SELECT COUNT(*) as count FROM salaryProfileEmployee + WHERE salaryOrgId IN ( + SELECT id FROM salaryOrgEmployee + WHERE salaryPeriodId = ? AND snapshot = ? + ) + `, + [salaryPeriod.id, snapshot], + ).then((result) => result[0]?.count || 0) + : Promise.resolve(0), + ]); + + console.log( + `🧹 ลบข้อมูลเก่า: SalaryProfile=${salaryProfileCount}, SalaryProfileEmployee=${salaryProfileEmployeeCount}, SalaryOrg=${salaryOrgCount}, SalaryOrgEmployee=${salaryOrgEmployeeCount}`, + ); + + // Fast bulk delete - ลบโดยใช้ query โดยตรง (เร็วกว่า 10-50 เท่า) + // First, delete child records (profiles) using bulk delete + await Promise.all([ + salaryOrgCount > 0 + ? AppDataSource.query( + ` + DELETE FROM salaryProfile + WHERE salaryOrgId IN ( + SELECT id FROM salaryOrg + WHERE salaryPeriodId = ? AND snapshot = ? + ) + `, + [salaryPeriod.id, snapshot], + ) + : Promise.resolve(), + salaryOrgEmployeeCount > 0 + ? AppDataSource.query( + ` + DELETE FROM salaryProfileEmployee + WHERE salaryOrgId IN ( + SELECT id FROM salaryOrgEmployee + WHERE salaryPeriodId = ? AND snapshot = ? + ) + `, + [salaryPeriod.id, snapshot], + ) + : Promise.resolve(), + ]); + + // Then, delete parent records (orgs) + await Promise.all([ + AppDataSource.query( + ` + DELETE FROM salaryOrg + WHERE salaryPeriodId = ? AND snapshot = ? + `, + [salaryPeriod.id, snapshot], + ), + AppDataSource.query( + ` + DELETE FROM salaryOrgEmployee + WHERE salaryPeriodId = ? AND snapshot = ? + `, + [salaryPeriod.id, snapshot], + ), + ]); + } else { + console.log(`ℹ️ ไม่มีข้อมูลที่ต้องลบสำหรับ ${snapshot}`); + } + } + + console.timeEnd("⏱ Cleanup: Fast bulk delete"); //snap บางสำนัก //.chin @@ -2797,11 +2979,27 @@ export class SalaryPeriodController extends Controller { } }); - // Batch insert SalaryOrg + // Batch insert SalaryOrg with error handling console.time("⏱ Insert: SalaryOrg batch"); if (salaryOrgsToSave.length > 0) { - await this.salaryOrgRepository.save(salaryOrgsToSave, { data: request }); - console.log(`✅ [SNAP: ${snapshot}] บันทึก ${salaryOrgsToSave.length} SalaryOrg สำเร็จ`); + try { + await this.salaryOrgRepository.save(salaryOrgsToSave, { data: request }); + console.log(`✅ [SNAP: ${snapshot}] บันทึก ${salaryOrgsToSave.length} SalaryOrg สำเร็จ`); + } catch (saveError) { + console.error("❌ Error saving SalaryOrg batch, trying chunked approach:", saveError); + // Retry with smaller chunks if failed + const chunkSize = 50; + for (let i = 0; i < salaryOrgsToSave.length; i += chunkSize) { + const chunk = salaryOrgsToSave.slice(i, i + chunkSize); + await this.salaryOrgRepository.save(chunk, { data: request }); + console.log( + `✅ Saved SalaryOrg chunk ${Math.floor(i / chunkSize) + 1}/${Math.ceil(salaryOrgsToSave.length / chunkSize)}`, + ); + + // Small delay to prevent overwhelming the connection + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } } console.timeEnd("⏱ Insert: SalaryOrg batch"); @@ -2836,13 +3034,32 @@ export class SalaryPeriodController extends Controller { } }); - // Batch insert SalaryOrgEmployee + // Batch insert SalaryOrgEmployee with error handling console.time("⏱ Insert: SalaryOrgEmployee batch"); if (salaryOrgEmployeesToSave.length > 0) { - await this.salaryOrgEmployeeRepository.save(salaryOrgEmployeesToSave, { data: request }); - console.log( - `✅ [SNAP: ${snapshot}] บันทึก ${salaryOrgEmployeesToSave.length} SalaryOrgEmployee สำเร็จ`, - ); + try { + await this.salaryOrgEmployeeRepository.save(salaryOrgEmployeesToSave, { data: request }); + console.log( + `✅ [SNAP: ${snapshot}] บันทึก ${salaryOrgEmployeesToSave.length} SalaryOrgEmployee สำเร็จ`, + ); + } catch (saveError) { + console.error( + "❌ Error saving SalaryOrgEmployee batch, trying chunked approach:", + saveError, + ); + // Retry with smaller chunks if failed + const chunkSize = 50; + for (let i = 0; i < salaryOrgEmployeesToSave.length; i += chunkSize) { + const chunk = salaryOrgEmployeesToSave.slice(i, i + chunkSize); + await this.salaryOrgEmployeeRepository.save(chunk, { data: request }); + console.log( + `✅ Saved SalaryOrgEmployee chunk ${Math.floor(i / chunkSize) + 1}/${Math.ceil(salaryOrgEmployeesToSave.length / chunkSize)}`, + ); + + // Small delay to prevent overwhelming the connection + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } } console.timeEnd("⏱ Insert: SalaryOrgEmployee batch"); if (salaryPeriod.period != "SPECIAL") { @@ -2925,9 +3142,9 @@ export class SalaryPeriodController extends Controller { console.timeEnd("⏱ Step3: Process Profiles"); - // Batch insert SalaryProfile with chunking for memory efficiency + // Batch insert SalaryProfile with chunking for memory efficiency and connection management console.time("⏱ Step4: Save All Profiles (Chunked)"); - const chunkSize = 500; // Reduced chunk size to prevent timeout + const chunkSize = 300; // Reduced chunk size to prevent timeout and SSL errors for (let i = 0; i < allProfilesToSave.length; i += chunkSize) { const chunk = allProfilesToSave.slice(i, i + chunkSize); try { @@ -2939,13 +3156,27 @@ export class SalaryPeriodController extends Controller { .orIgnore() .execute(); - // Progress logging every 10 chunks + // Progress logging every 10 chunks and small delay to prevent connection overload if ((i / chunkSize) % 10 === 0) { console.log(`📝 บันทึก SalaryProfile: ${i + chunk.length}/${allProfilesToSave.length}`); + // Small delay to prevent SSL connection issues + await new Promise((resolve) => setTimeout(resolve, 50)); } } catch (chunkErr) { console.error(`❌ Error saving SalaryProfile chunk ${i}-${i + chunk.length}:`, chunkErr); - throw chunkErr; + // Retry this chunk with even smaller size + const retryChunkSize = 50; + for (let j = 0; j < chunk.length; j += retryChunkSize) { + const retryChunk = chunk.slice(j, j + retryChunkSize); + await this.salaryProfileRepository + .createQueryBuilder() + .insert() + .into(SalaryProfile) + .values(retryChunk) + .orIgnore() + .execute(); + await new Promise((resolve) => setTimeout(resolve, 100)); + } } } console.timeEnd("⏱ Step4: Save All Profiles (Chunked)"); @@ -3032,10 +3263,10 @@ export class SalaryPeriodController extends Controller { } console.timeEnd("⏱ Step5: Process ProfileEmps"); - // Batch insert SalaryProfileEmployee with chunking + // Batch insert SalaryProfileEmployee with chunking and error recovery console.time("⏱ Step6: Save ProfileEmployees (Chunked)"); if (profilesEmpToSave.length > 0) { - const chunkSize = 500; // Reduced chunk size for better stability + const chunkSize = 300; // Reduced chunk size for better stability and SSL handling for (let i = 0; i < profilesEmpToSave.length; i += chunkSize) { const chunk = profilesEmpToSave.slice(i, i + chunkSize); try { @@ -3047,18 +3278,32 @@ export class SalaryPeriodController extends Controller { .orIgnore() .execute(); - // Progress logging every 10 chunks + // Progress logging every 10 chunks with small delay if ((i / chunkSize) % 10 === 0) { console.log( `📝 บันทึก SalaryProfileEmployee: ${i + chunk.length}/${profilesEmpToSave.length}`, ); + // Small delay to prevent SSL connection issues + await new Promise((resolve) => setTimeout(resolve, 50)); } } catch (chunkErr) { console.error( `❌ Error saving SalaryProfileEmployee chunk ${i}-${i + chunk.length}:`, chunkErr, ); - throw chunkErr; + // Retry this chunk with even smaller size + const retryChunkSize = 50; + for (let j = 0; j < chunk.length; j += retryChunkSize) { + const retryChunk = chunk.slice(j, j + retryChunkSize); + await this.salaryProfileEmployeeRepository + .createQueryBuilder() + .insert() + .into(SalaryProfileEmployee) + .values(retryChunk) + .orIgnore() + .execute(); + await new Promise((resolve) => setTimeout(resolve, 100)); + } } } console.log(