Merge branch 'develop' into forgejoDev

This commit is contained in:
AdisakKanthawilang 2025-09-30 15:38:31 +07:00
commit 12f3f7066f

View file

@ -2563,11 +2563,16 @@ export class SalaryPeriodController extends Controller {
errorMessage,
);
// Check if it's a connection error
// Check if it's a connection or SSL error
if (
errorMessage.includes("ECONNRESET") ||
errorMessage.includes("Connection lost") ||
errorMessage.includes("ETIMEDOUT")
errorMessage.includes("ETIMEDOUT") ||
errorMessage.includes("SSL routines") ||
errorMessage.includes("decryption failed") ||
errorMessage.includes("bad record mac") ||
errorMessage.includes("EPIPE") ||
errorMessage.includes("socket hang up")
) {
retryCount++;
if (retryCount < maxRetries) {
@ -2599,6 +2604,22 @@ export class SalaryPeriodController extends Controller {
salaryPeriodId: string,
request: RequestWithUser,
) {
// ปรับ connection timeout เพื่อรองรับการประมวลผลข้อมูลขนาดใหญ่
const connection = AppDataSource;
if (connection.isInitialized) {
try {
// ปรับ query timeout สำหรับ long-running operations
await connection.query("SET SESSION wait_timeout = 3600"); // 1 hour
await connection.query("SET SESSION interactive_timeout = 3600");
await connection.query("SET SESSION max_execution_time = 0"); // No limit
await connection.query("SET SESSION net_read_timeout = 600"); // 10 minutes
await connection.query("SET SESSION net_write_timeout = 600"); // 10 minutes
console.log("✅ ปรับ database session timeout เรียบร้อย");
} catch (sessionErr) {
console.warn("⚠️ ไม่สามารถปรับ session timeout ได้:", sessionErr);
}
}
const salaryPeriod = await this.salaryPeriodRepository.findOne({
where: { id: salaryPeriodId },
});
@ -2606,60 +2627,221 @@ export class SalaryPeriodController extends Controller {
throw new HttpError(HttpStatusCode.NOT_FOUND, "ไม่พบรอบการสร้างเงินเดือน");
}
// Parallel loading of existing data for removal
console.time("⏱ Cleanup: Load existing data");
const [salaryOrg, salaryOrgEmployee] = await Promise.all([
this.salaryOrgRepository.find({
where: { salaryPeriodId: salaryPeriod.id, snapshot: snapshot },
}),
this.salaryOrgEmployeeRepository.find({
where: { salaryPeriodId: salaryPeriod.id, snapshot: snapshot },
}),
]);
// Fast bulk delete approach - ไม่ต้อง load ข้อมูลมาก่อนลบ
console.time("⏱ Cleanup: Fast bulk delete");
// Parallel loading of profiles for removal
const [salaryProfile, salaryProfileEmployee] = await Promise.all([
salaryOrg.length > 0
? this.salaryProfileRepository.find({
where: { salaryOrgId: In(salaryOrg.map((x) => x.id)) },
})
: Promise.resolve([]),
salaryOrgEmployee.length > 0
? this.salaryProfileEmployeeRepository.find({
where: { salaryOrgId: In(salaryOrgEmployee.map((x) => x.id)) },
})
: Promise.resolve([]),
]);
console.timeEnd("⏱ Cleanup: Load existing data");
// สำหรับข้อมูลจำนวนมาก ใช้ TRUNCATE หรือ bulk delete
// Option 1: ถ้าต้องการลบข้อมูลทั้งหมดของ snapshot
if (snapshot === "SNAP1" || snapshot === "SNAP2") {
console.log(`🚀 ใช้ Super Fast Delete สำหรับ ${snapshot}`);
console.log(
`🧹 ลบข้อมูลเก่า: SalaryProfile=${salaryProfile.length}, SalaryProfileEmployee=${salaryProfileEmployee.length}, SalaryOrg=${salaryOrg.length}, SalaryOrgEmployee=${salaryOrgEmployee.length}`,
);
// Super fast approach: ลบแบบ sequential เพื่อเคารพ foreign key constraints
// แต่ใช้ bulk delete แทนการ load + remove
// Sequential removal to respect foreign key constraints
// First, remove all profiles (child records)
console.time("⏱ Cleanup: Remove child records");
await Promise.all([
salaryProfile.length > 0
? this.salaryProfileRepository.remove(salaryProfile, { data: request })
: Promise.resolve(),
salaryProfileEmployee.length > 0
? this.salaryProfileEmployeeRepository.remove(salaryProfileEmployee, { data: request })
: Promise.resolve(),
]);
console.timeEnd("⏱ Cleanup: Remove child records");
// For extremely large datasets (100K+ records), use batch delete to prevent timeouts
const isLargeDataset = await AppDataSource.query(
`
SELECT COUNT(*) as count FROM salaryOrg WHERE salaryPeriodId = ? AND snapshot = ?
`,
[salaryPeriod.id, snapshot],
).then((result) => result[0]?.count > 50000);
// Then, remove the org records (parent records)
console.time("⏱ Cleanup: Remove parent records");
await Promise.all([
salaryOrg.length > 0
? this.salaryOrgRepository.remove(salaryOrg, { data: request })
: Promise.resolve(),
salaryOrgEmployee.length > 0
? this.salaryOrgEmployeeRepository.remove(salaryOrgEmployee, { data: request })
: Promise.resolve(),
]);
console.timeEnd("⏱ Cleanup: Remove parent records");
if (isLargeDataset) {
console.log(`🔥 ข้อมูลขนาดใหญ่มาก - ใช้ Batch Delete`);
// Batch delete for large datasets to prevent timeout
let deletedProfiles = 0,
deletedProfileEmployees = 0;
// Delete profiles in batches
while (true) {
const result = await AppDataSource.query(
`
DELETE sp FROM salaryProfile sp
INNER JOIN salaryOrg so ON sp.salaryOrgId = so.id
WHERE so.salaryPeriodId = ? AND so.snapshot = ?
LIMIT 10000
`,
[salaryPeriod.id, snapshot],
);
deletedProfiles += result.affectedRows;
if (result.affectedRows === 0) break;
// Small delay to prevent overwhelming the database
await new Promise((resolve) => setTimeout(resolve, 100));
}
// Delete profile employees in batches
while (true) {
const result = await AppDataSource.query(
`
DELETE spe FROM salaryProfileEmployee spe
INNER JOIN salaryOrgEmployee soe ON spe.salaryOrgId = soe.id
WHERE soe.salaryPeriodId = ? AND soe.snapshot = ?
LIMIT 10000
`,
[salaryPeriod.id, snapshot],
);
deletedProfileEmployees += result.affectedRows;
if (result.affectedRows === 0) break;
await new Promise((resolve) => setTimeout(resolve, 100));
}
// Delete orgs and org employees
const [deletedOrg, deletedOrgEmployee] = await Promise.all([
AppDataSource.query(`DELETE FROM salaryOrg WHERE salaryPeriodId = ? AND snapshot = ?`, [
salaryPeriod.id,
snapshot,
]),
AppDataSource.query(
`DELETE FROM salaryOrgEmployee WHERE salaryPeriodId = ? AND snapshot = ?`,
[salaryPeriod.id, snapshot],
),
]);
console.log(
`✅ Batch Delete เสร็จสิ้น - Profiles: ${deletedProfiles}, ProfileEmployees: ${deletedProfileEmployees}, Orgs: ${deletedOrg.affectedRows}, OrgEmployees: ${deletedOrgEmployee.affectedRows}`,
);
} else {
// Standard fast delete for smaller datasets
// Step 1: ลบ SalaryProfile ก่อน
const deletedProfiles = await AppDataSource.query(
`
DELETE sp FROM salaryProfile sp
INNER JOIN salaryOrg so ON sp.salaryOrgId = so.id
WHERE so.salaryPeriodId = ? AND so.snapshot = ?
`,
[salaryPeriod.id, snapshot],
);
// Step 2: ลบ SalaryProfileEmployee
const deletedProfileEmployees = await AppDataSource.query(
`
DELETE spe FROM salaryProfileEmployee spe
INNER JOIN salaryOrgEmployee soe ON spe.salaryOrgId = soe.id
WHERE soe.salaryPeriodId = ? AND soe.snapshot = ?
`,
[salaryPeriod.id, snapshot],
);
// Step 3: ลบ SalaryOrg และ SalaryOrgEmployee
const [deletedOrg, deletedOrgEmployee] = await Promise.all([
AppDataSource.query(`DELETE FROM salaryOrg WHERE salaryPeriodId = ? AND snapshot = ?`, [
salaryPeriod.id,
snapshot,
]),
AppDataSource.query(
`DELETE FROM salaryOrgEmployee WHERE salaryPeriodId = ? AND snapshot = ?`,
[salaryPeriod.id, snapshot],
),
]);
console.log(
`✅ Super Fast Delete เสร็จสิ้น - Profiles: ${deletedProfiles.affectedRows}, ProfileEmployees: ${deletedProfileEmployees.affectedRows}, Orgs: ${deletedOrg.affectedRows}, OrgEmployees: ${deletedOrgEmployee.affectedRows}`,
);
}
} else {
// Fallback: Standard bulk delete approach สำหรับ snapshots อื่นๆ
console.log(`📊 ใช้ Standard Bulk Delete สำหรับ ${snapshot}`);
// Get counts for logging (optional, can be removed for even better performance)
const [salaryOrgCount, salaryOrgEmployeeCount] = await Promise.all([
this.salaryOrgRepository.count({
where: { salaryPeriodId: salaryPeriod.id, snapshot: snapshot },
}),
this.salaryOrgEmployeeRepository.count({
where: { salaryPeriodId: salaryPeriod.id, snapshot: snapshot },
}),
]);
if (salaryOrgCount > 0 || salaryOrgEmployeeCount > 0) {
// Get profile counts using subqueries
const [salaryProfileCount, salaryProfileEmployeeCount] = await Promise.all([
salaryOrgCount > 0
? AppDataSource.query(
`
SELECT COUNT(*) as count FROM salaryProfile
WHERE salaryOrgId IN (
SELECT id FROM salaryOrg
WHERE salaryPeriodId = ? AND snapshot = ?
)
`,
[salaryPeriod.id, snapshot],
).then((result) => result[0]?.count || 0)
: Promise.resolve(0),
salaryOrgEmployeeCount > 0
? AppDataSource.query(
`
SELECT COUNT(*) as count FROM salaryProfileEmployee
WHERE salaryOrgId IN (
SELECT id FROM salaryOrgEmployee
WHERE salaryPeriodId = ? AND snapshot = ?
)
`,
[salaryPeriod.id, snapshot],
).then((result) => result[0]?.count || 0)
: Promise.resolve(0),
]);
console.log(
`🧹 ลบข้อมูลเก่า: SalaryProfile=${salaryProfileCount}, SalaryProfileEmployee=${salaryProfileEmployeeCount}, SalaryOrg=${salaryOrgCount}, SalaryOrgEmployee=${salaryOrgEmployeeCount}`,
);
// Fast bulk delete - ลบโดยใช้ query โดยตรง (เร็วกว่า 10-50 เท่า)
// First, delete child records (profiles) using bulk delete
await Promise.all([
salaryOrgCount > 0
? AppDataSource.query(
`
DELETE FROM salaryProfile
WHERE salaryOrgId IN (
SELECT id FROM salaryOrg
WHERE salaryPeriodId = ? AND snapshot = ?
)
`,
[salaryPeriod.id, snapshot],
)
: Promise.resolve(),
salaryOrgEmployeeCount > 0
? AppDataSource.query(
`
DELETE FROM salaryProfileEmployee
WHERE salaryOrgId IN (
SELECT id FROM salaryOrgEmployee
WHERE salaryPeriodId = ? AND snapshot = ?
)
`,
[salaryPeriod.id, snapshot],
)
: Promise.resolve(),
]);
// Then, delete parent records (orgs)
await Promise.all([
AppDataSource.query(
`
DELETE FROM salaryOrg
WHERE salaryPeriodId = ? AND snapshot = ?
`,
[salaryPeriod.id, snapshot],
),
AppDataSource.query(
`
DELETE FROM salaryOrgEmployee
WHERE salaryPeriodId = ? AND snapshot = ?
`,
[salaryPeriod.id, snapshot],
),
]);
} else {
console.log(` ไม่มีข้อมูลที่ต้องลบสำหรับ ${snapshot}`);
}
}
console.timeEnd("⏱ Cleanup: Fast bulk delete");
//snap บางสำนัก
//.chin
@ -2797,11 +2979,27 @@ export class SalaryPeriodController extends Controller {
}
});
// Batch insert SalaryOrg
// Batch insert SalaryOrg with error handling
console.time("⏱ Insert: SalaryOrg batch");
if (salaryOrgsToSave.length > 0) {
await this.salaryOrgRepository.save(salaryOrgsToSave, { data: request });
console.log(`✅ [SNAP: ${snapshot}] บันทึก ${salaryOrgsToSave.length} SalaryOrg สำเร็จ`);
try {
await this.salaryOrgRepository.save(salaryOrgsToSave, { data: request });
console.log(`✅ [SNAP: ${snapshot}] บันทึก ${salaryOrgsToSave.length} SalaryOrg สำเร็จ`);
} catch (saveError) {
console.error("❌ Error saving SalaryOrg batch, trying chunked approach:", saveError);
// Retry with smaller chunks if failed
const chunkSize = 50;
for (let i = 0; i < salaryOrgsToSave.length; i += chunkSize) {
const chunk = salaryOrgsToSave.slice(i, i + chunkSize);
await this.salaryOrgRepository.save(chunk, { data: request });
console.log(
`✅ Saved SalaryOrg chunk ${Math.floor(i / chunkSize) + 1}/${Math.ceil(salaryOrgsToSave.length / chunkSize)}`,
);
// Small delay to prevent overwhelming the connection
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
}
console.timeEnd("⏱ Insert: SalaryOrg batch");
@ -2836,13 +3034,32 @@ export class SalaryPeriodController extends Controller {
}
});
// Batch insert SalaryOrgEmployee
// Batch insert SalaryOrgEmployee with error handling
console.time("⏱ Insert: SalaryOrgEmployee batch");
if (salaryOrgEmployeesToSave.length > 0) {
await this.salaryOrgEmployeeRepository.save(salaryOrgEmployeesToSave, { data: request });
console.log(
`✅ [SNAP: ${snapshot}] บันทึก ${salaryOrgEmployeesToSave.length} SalaryOrgEmployee สำเร็จ`,
);
try {
await this.salaryOrgEmployeeRepository.save(salaryOrgEmployeesToSave, { data: request });
console.log(
`✅ [SNAP: ${snapshot}] บันทึก ${salaryOrgEmployeesToSave.length} SalaryOrgEmployee สำเร็จ`,
);
} catch (saveError) {
console.error(
"❌ Error saving SalaryOrgEmployee batch, trying chunked approach:",
saveError,
);
// Retry with smaller chunks if failed
const chunkSize = 50;
for (let i = 0; i < salaryOrgEmployeesToSave.length; i += chunkSize) {
const chunk = salaryOrgEmployeesToSave.slice(i, i + chunkSize);
await this.salaryOrgEmployeeRepository.save(chunk, { data: request });
console.log(
`✅ Saved SalaryOrgEmployee chunk ${Math.floor(i / chunkSize) + 1}/${Math.ceil(salaryOrgEmployeesToSave.length / chunkSize)}`,
);
// Small delay to prevent overwhelming the connection
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
}
console.timeEnd("⏱ Insert: SalaryOrgEmployee batch");
if (salaryPeriod.period != "SPECIAL") {
@ -2925,9 +3142,9 @@ export class SalaryPeriodController extends Controller {
console.timeEnd("⏱ Step3: Process Profiles");
// Batch insert SalaryProfile with chunking for memory efficiency
// Batch insert SalaryProfile with chunking for memory efficiency and connection management
console.time("⏱ Step4: Save All Profiles (Chunked)");
const chunkSize = 500; // Reduced chunk size to prevent timeout
const chunkSize = 300; // Reduced chunk size to prevent timeout and SSL errors
for (let i = 0; i < allProfilesToSave.length; i += chunkSize) {
const chunk = allProfilesToSave.slice(i, i + chunkSize);
try {
@ -2939,13 +3156,27 @@ export class SalaryPeriodController extends Controller {
.orIgnore()
.execute();
// Progress logging every 10 chunks
// Progress logging every 10 chunks and small delay to prevent connection overload
if ((i / chunkSize) % 10 === 0) {
console.log(`📝 บันทึก SalaryProfile: ${i + chunk.length}/${allProfilesToSave.length}`);
// Small delay to prevent SSL connection issues
await new Promise((resolve) => setTimeout(resolve, 50));
}
} catch (chunkErr) {
console.error(`❌ Error saving SalaryProfile chunk ${i}-${i + chunk.length}:`, chunkErr);
throw chunkErr;
// Retry this chunk with even smaller size
const retryChunkSize = 50;
for (let j = 0; j < chunk.length; j += retryChunkSize) {
const retryChunk = chunk.slice(j, j + retryChunkSize);
await this.salaryProfileRepository
.createQueryBuilder()
.insert()
.into(SalaryProfile)
.values(retryChunk)
.orIgnore()
.execute();
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
}
console.timeEnd("⏱ Step4: Save All Profiles (Chunked)");
@ -3032,10 +3263,10 @@ export class SalaryPeriodController extends Controller {
}
console.timeEnd("⏱ Step5: Process ProfileEmps");
// Batch insert SalaryProfileEmployee with chunking
// Batch insert SalaryProfileEmployee with chunking and error recovery
console.time("⏱ Step6: Save ProfileEmployees (Chunked)");
if (profilesEmpToSave.length > 0) {
const chunkSize = 500; // Reduced chunk size for better stability
const chunkSize = 300; // Reduced chunk size for better stability and SSL handling
for (let i = 0; i < profilesEmpToSave.length; i += chunkSize) {
const chunk = profilesEmpToSave.slice(i, i + chunkSize);
try {
@ -3047,18 +3278,32 @@ export class SalaryPeriodController extends Controller {
.orIgnore()
.execute();
// Progress logging every 10 chunks
// Progress logging every 10 chunks with small delay
if ((i / chunkSize) % 10 === 0) {
console.log(
`📝 บันทึก SalaryProfileEmployee: ${i + chunk.length}/${profilesEmpToSave.length}`,
);
// Small delay to prevent SSL connection issues
await new Promise((resolve) => setTimeout(resolve, 50));
}
} catch (chunkErr) {
console.error(
`❌ Error saving SalaryProfileEmployee chunk ${i}-${i + chunk.length}:`,
chunkErr,
);
throw chunkErr;
// Retry this chunk with even smaller size
const retryChunkSize = 50;
for (let j = 0; j < chunk.length; j += retryChunkSize) {
const retryChunk = chunk.slice(j, j + retryChunkSize);
await this.salaryProfileEmployeeRepository
.createQueryBuilder()
.insert()
.into(SalaryProfileEmployee)
.values(retryChunk)
.orIgnore()
.execute();
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
}
console.log(