hrms-bangkok/hrms-edm
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

feat: cache metadata on requeue

Browse source
This commit is contained in:
Methapon2001 2023-11-24 18:04:36 +07:00
parent 9ca1213e50
commit 0a9f27a02e
No known key found for this signature in database
GPG key ID: 849924FEF46BD132
1 changed files with 32 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

43
Services/server/src/rabbitmq/handler.ts
View file

@ -4,6 +4,7 @@ import minioClient from "../storage";
// for failed queue that will come later // for failed queue that will come later
const cachedBuffer: Record<string, Buffer> = {}; const cachedBuffer: Record<string, Buffer> = {};
const cachedMetadata: Record<string, { size: number; type: string }> = {};
export async function handler(key: string): Promise<boolean> { export async function handler(key: string): Promise<boolean> {
console.info(`[AMQ] Messages received - key: ${key}`); console.info(`[AMQ] Messages received - key: ${key}`);
@ -17,19 +18,27 @@ export async function handler(key: string): Promise<boolean> {
cachedBuffer[key] = buffer; cachedBuffer[key] = buffer;
} }
const rec = await getInfo(pathname); if (!cachedMetadata[key]) {
const stat = await minioClient.statObject(bucket, pathname);
cachedMetadata[key] = { size: stat.size, type: stat.metaData["content-type"] };
}
const rec = await popInfo(pathname);
const result = rec const result = rec
? await handleFoundRecord(rec, cachedBuffer[key]) ? await handleFoundRecord(rec, cachedBuffer[key], cachedMetadata[key])
: await handleNotFoundRecord(pathname, cachedBuffer[key]); : await handleNotFoundRecord(pathname, cachedBuffer[key], cachedMetadata[key]);
if (result) delete cachedBuffer[key]; if (result) {
delete cachedBuffer[key];
delete cachedMetadata[key];
}
return result; return result;
} }
// Get info and delete it from ElasticSearch to re-index // Get info and delete it from ElasticSearch to re-index
async function getInfo(pathname: string) { async function popInfo(pathname: string) {
const result = await esClient const result = await esClient
.search<EhrFile & { attachment?: Record<string, unknown> }>({ .search<EhrFile & { attachment?: Record<string, unknown> }>({
index: "my-test-index", index: "my-test-index",
@ -58,15 +67,19 @@ async function getInfo(pathname: string) {
* Handle when record in elasticsearch cannot be found. * Handle when record in elasticsearch cannot be found.
* This will insert empty metadata. * This will insert empty metadata.
*/ */
async function handleNotFoundRecord(pathname: string, buffer: Buffer) { async function handleNotFoundRecord(
pathname: string,
buffer: Buffer,
stat: { size: number; type: string },
) {
const filename = pathname.split("/").at(-1); const filename = pathname.split("/").at(-1);
const base64 = Buffer.from(buffer).toString("base64"); const base64 = Buffer.from(buffer).toString("base64");
const metadata = { const metadata = {
pathname, pathname,
fileName: filename ?? "n/a", // should not possible to fallback fileName: filename ?? "n/a", // should not possible to fallback, just in case.
fileSize: Buffer.byteLength(buffer), fileSize: stat.size,
fileType: "", fileType: stat.type,
title: "", title: "",
description: "", description: "",
category: [], category: [],
@ -91,12 +104,20 @@ async function handleNotFoundRecord(pathname: string, buffer: Buffer) {
return false; return false;
} }
async function handleFoundRecord(metadata: EhrFile, buffer: Buffer) { async function handleFoundRecord(
metadata: EhrFile,
buffer: Buffer,
stat: { size: number; type: string },
) {
metadata.fileSize = stat.size;
metadata.fileType = stat.type;
metadata.upload = true;
const result = await esClient const result = await esClient
.index({ .index({
pipeline: "attachment", pipeline: "attachment",
index: "my-test-index", index: "my-test-index",
document: { data: Buffer.from(buffer).toString("base64"), ...metadata, upload: true }, document: { data: Buffer.from(buffer).toString("base64"), ...metadata },
}) })
.catch((e) => console.error(e)); .catch((e) => console.error(e));