Production readiness hardening and ops tooling
This commit is contained in:
413
scripts/load-test.mjs
Normal file
413
scripts/load-test.mjs
Normal file
@ -0,0 +1,413 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHmac } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { Pool } from "pg";
|
||||
import "dotenv/config";
|
||||
|
||||
const PORT = process.env.PORT || "3120";
|
||||
const BASE = process.env.BASE_URL || `http://127.0.0.1:${PORT}`;
|
||||
const ADMIN_TOKEN = process.env.ADMIN_TOKEN || "admin-dev-token";
|
||||
const DEVICE_TOKEN = process.env.DEVICE_TOKEN || "device-dev-token";
|
||||
const SECRET = process.env.INTEGRATION_WEBHOOK_SECRET || "dev-callback-secret";
|
||||
|
||||
const RUN_ID = process.env.LOAD_RUN_ID || `load-${Date.now()}`;
|
||||
const CALLBACKS = Number(process.env.LOAD_CALLBACKS || 30);
|
||||
const HEARTBEATS = Number(process.env.LOAD_HEARTBEATS || 60);
|
||||
const DYNAMIC_QR = Number(process.env.LOAD_DYNAMIC_QR || 30);
|
||||
const READS = Number(process.env.LOAD_READS || 30);
|
||||
const EXPORTS = Number(process.env.LOAD_EXPORTS || 0);
|
||||
const CONCURRENCY = Number(process.env.LOAD_CONCURRENCY || 10);
|
||||
|
||||
const created = {
|
||||
merchantIds: [],
|
||||
partnerReferences: []
|
||||
};
|
||||
|
||||
const metrics = new Map();
|
||||
|
||||
function percentile(values, p) {
|
||||
if (!values.length) {
|
||||
return 0;
|
||||
}
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const index = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
|
||||
return sorted[index];
|
||||
}
|
||||
|
||||
function record(label, durationMs, ok) {
|
||||
const entry = metrics.get(label) || { durations: [], ok: 0, error: 0 };
|
||||
entry.durations.push(durationMs);
|
||||
if (ok) {
|
||||
entry.ok += 1;
|
||||
} else {
|
||||
entry.error += 1;
|
||||
}
|
||||
metrics.set(label, entry);
|
||||
}
|
||||
|
||||
async function req(path, options = {}) {
|
||||
const startedAt = performance.now();
|
||||
const label = options.label || `${options.method || "GET"} ${path}`;
|
||||
try {
|
||||
const response = await fetch(`${BASE}${path}`, {
|
||||
method: options.method || "GET",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(options.headers || {})
|
||||
},
|
||||
body: Object.prototype.hasOwnProperty.call(options, "body") ? JSON.stringify(options.body) : undefined
|
||||
});
|
||||
const text = await response.text();
|
||||
let body = null;
|
||||
try {
|
||||
body = text ? JSON.parse(text) : null;
|
||||
} catch {
|
||||
body = text;
|
||||
}
|
||||
const ok = response.ok;
|
||||
record(label, performance.now() - startedAt, ok);
|
||||
if (!ok) {
|
||||
throw new Error(`${label} failed status=${response.status} body=${typeof body === "string" ? body.slice(0, 120) : JSON.stringify(body).slice(0, 120)}`);
|
||||
}
|
||||
return body?.data !== undefined ? body.data : body;
|
||||
} catch (error) {
|
||||
record(label, performance.now() - startedAt, false);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function reqAdmin(path, options = {}) {
|
||||
return req(path, {
|
||||
...options,
|
||||
headers: {
|
||||
...(options.headers || {}),
|
||||
Authorization: `Bearer ${ADMIN_TOKEN}`
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function reqDevice(path, options = {}) {
|
||||
return req(path, {
|
||||
...options,
|
||||
headers: {
|
||||
...(options.headers || {}),
|
||||
Authorization: `Bearer ${DEVICE_TOKEN}`
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function runPool(items, concurrency, worker) {
|
||||
const queue = [...items];
|
||||
const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
|
||||
while (queue.length) {
|
||||
const item = queue.shift();
|
||||
await worker(item);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
}
|
||||
|
||||
async function waitForHealth() {
|
||||
for (let i = 0; i < 100; i += 1) {
|
||||
try {
|
||||
await req("/health", { label: "health_check" });
|
||||
return;
|
||||
} catch {
|
||||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||
}
|
||||
}
|
||||
throw new Error("health check timeout");
|
||||
}
|
||||
|
||||
async function createBundle() {
|
||||
const merchant = await reqAdmin("/admin/merchants", {
|
||||
method: "POST",
|
||||
body: {
|
||||
legal_name: `Load Test Merchant ${RUN_ID}`,
|
||||
brand_name: `LOAD-${RUN_ID}`,
|
||||
settlement_account_reference: `bank:${RUN_ID}`,
|
||||
settlement_account_type: "merchant_bank_account",
|
||||
payout_mode: "merchant_direct"
|
||||
},
|
||||
label: "setup_create_merchant"
|
||||
});
|
||||
created.merchantIds.push(merchant.id);
|
||||
|
||||
const outlet = await reqAdmin(`/admin/merchants/${merchant.id}/outlets`, {
|
||||
method: "POST",
|
||||
body: { name: `Load Outlet ${RUN_ID}` },
|
||||
label: "setup_create_outlet"
|
||||
});
|
||||
const terminal = await reqAdmin(`/admin/outlets/${outlet.id}/terminals`, {
|
||||
method: "POST",
|
||||
body: { terminal_code: `LOAD-TERM-${RUN_ID}`, qr_mode: "static" },
|
||||
label: "setup_create_terminal"
|
||||
});
|
||||
const device = await reqAdmin("/admin/devices", {
|
||||
method: "POST",
|
||||
body: {
|
||||
device_code: `LOAD-DEV-${RUN_ID}`,
|
||||
vendor: "load",
|
||||
model: "static",
|
||||
communication_mode: "mqtt",
|
||||
status: "active"
|
||||
},
|
||||
label: "setup_create_device"
|
||||
});
|
||||
await reqAdmin(`/admin/devices/${device.id}/bind`, {
|
||||
method: "POST",
|
||||
body: {
|
||||
merchant_id: merchant.id,
|
||||
outlet_id: outlet.id,
|
||||
terminal_id: terminal.id
|
||||
},
|
||||
label: "setup_bind_device"
|
||||
});
|
||||
|
||||
const dynamicOutlet = await reqAdmin(`/admin/merchants/${merchant.id}/outlets`, {
|
||||
method: "POST",
|
||||
body: { name: `Load Dynamic Outlet ${RUN_ID}` },
|
||||
label: "setup_create_dynamic_outlet"
|
||||
});
|
||||
const dynamicTerminal = await reqAdmin(`/admin/outlets/${dynamicOutlet.id}/terminals`, {
|
||||
method: "POST",
|
||||
body: { terminal_code: `LOAD-DYN-${RUN_ID}`, qr_mode: "dynamic_api" },
|
||||
label: "setup_create_dynamic_terminal"
|
||||
});
|
||||
const dynamicDevice = await reqAdmin("/admin/devices", {
|
||||
method: "POST",
|
||||
body: {
|
||||
device_code: `LOAD-DYN-DEV-${RUN_ID}`,
|
||||
vendor: "load",
|
||||
model: "dynamic-api",
|
||||
communication_mode: "api",
|
||||
capability_profile_json: {
|
||||
dynamic_qr: { api_direct: true, mqtt: false },
|
||||
flows: ["dynamic_qr:api_direct", "static_payment_notification"]
|
||||
},
|
||||
status: "active"
|
||||
},
|
||||
label: "setup_create_dynamic_device"
|
||||
});
|
||||
await reqAdmin(`/admin/devices/${dynamicDevice.id}/bind`, {
|
||||
method: "POST",
|
||||
body: {
|
||||
merchant_id: merchant.id,
|
||||
outlet_id: dynamicOutlet.id,
|
||||
terminal_id: dynamicTerminal.id
|
||||
},
|
||||
label: "setup_bind_dynamic_device"
|
||||
});
|
||||
|
||||
return {
|
||||
merchant,
|
||||
outlet,
|
||||
terminal,
|
||||
device,
|
||||
dynamicOutlet,
|
||||
dynamicTerminal,
|
||||
dynamicDevice
|
||||
};
|
||||
}
|
||||
|
||||
async function createTransaction(bundle, index) {
|
||||
const partnerReference = `LOAD-PR-${RUN_ID}-${index}`;
|
||||
created.partnerReferences.push(partnerReference);
|
||||
return reqAdmin("/admin/transactions", {
|
||||
method: "POST",
|
||||
body: {
|
||||
partner_reference: partnerReference,
|
||||
merchant_id: bundle.merchant.id,
|
||||
outlet_id: bundle.outlet.id,
|
||||
terminal_id: bundle.terminal.id,
|
||||
device_id: bundle.device.id,
|
||||
amount: 10000 + index,
|
||||
currency: "IDR",
|
||||
qr_mode: "static",
|
||||
initiation_mode: "static",
|
||||
status: "initiated"
|
||||
},
|
||||
label: "transaction_create"
|
||||
});
|
||||
}
|
||||
|
||||
async function callbackPaid(index) {
|
||||
const callback = {
|
||||
partner_reference: `LOAD-PR-${RUN_ID}-${index}`,
|
||||
partner_txn_id: `LOAD-PTX-${RUN_ID}-${index}`,
|
||||
amount: 10000 + index,
|
||||
currency: "IDR",
|
||||
payment_status: "paid",
|
||||
status: "paid",
|
||||
paid_at: new Date().toISOString()
|
||||
};
|
||||
const signature = createHmac("sha256", SECRET).update(JSON.stringify(callback)).digest("hex");
|
||||
return req("/integrations/qris/callback", {
|
||||
method: "POST",
|
||||
headers: { "X-Partner-Signature": signature },
|
||||
body: { ...callback, signature },
|
||||
label: "qris_callback_paid"
|
||||
});
|
||||
}
|
||||
|
||||
async function heartbeat(bundle, index) {
|
||||
return reqDevice("/device/heartbeat", {
|
||||
method: "POST",
|
||||
body: {
|
||||
device_id: bundle.device.id,
|
||||
timestamp: new Date().toISOString(),
|
||||
firmware_version: "load-1.0.0",
|
||||
network_strength: 70 + (index % 20),
|
||||
battery_level: 60 + (index % 30),
|
||||
state: "load-test"
|
||||
},
|
||||
label: "device_heartbeat"
|
||||
});
|
||||
}
|
||||
|
||||
async function dynamicQr(bundle, index) {
|
||||
const requestId = `LOAD-DYN-${RUN_ID}-${index}`;
|
||||
return reqDevice("/device/transactions/dynamic-qr", {
|
||||
method: "POST",
|
||||
headers: { "Idempotency-Key": requestId },
|
||||
body: {
|
||||
device_id: bundle.dynamicDevice.id,
|
||||
terminal_id: bundle.dynamicTerminal.id,
|
||||
amount: 15000 + index,
|
||||
currency: "IDR",
|
||||
request_id: requestId,
|
||||
expires_in_seconds: 300
|
||||
},
|
||||
label: "dynamic_qr_create"
|
||||
});
|
||||
}
|
||||
|
||||
async function exportAdjustments(index) {
|
||||
const job = await reqAdmin("/admin/exports/settlement-adjustments", {
|
||||
method: "POST",
|
||||
body: { limit: 500 },
|
||||
label: "export_adjustments_create"
|
||||
});
|
||||
let current = job;
|
||||
for (let attempt = 0; attempt < 30 && !["completed", "failed"].includes(current.status); attempt += 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
current = await reqAdmin(`/admin/exports/${job.id}`, {
|
||||
label: "export_adjustments_poll"
|
||||
});
|
||||
}
|
||||
if (current.status !== "completed") {
|
||||
throw new Error(`export job ${index} ended with status ${current.status}`);
|
||||
}
|
||||
await reqAdmin(`/admin/exports/${job.id}/download`, {
|
||||
label: "export_adjustments_download"
|
||||
});
|
||||
}
|
||||
|
||||
async function cleanup() {
|
||||
const pool = new Pool(
|
||||
process.env.DATABASE_URL
|
||||
? { connectionString: process.env.DATABASE_URL }
|
||||
: {
|
||||
host: process.env.PGHOST || "127.0.0.1",
|
||||
port: Number(process.env.PGPORT || 5432),
|
||||
user: process.env.PGUSER || "postgres",
|
||||
password: process.env.PGPASSWORD || "postgres",
|
||||
database: process.env.PGDATABASE || "qris_soundbox_platform"
|
||||
}
|
||||
);
|
||||
try {
|
||||
const deletedTransactions = await pool.query(
|
||||
"DELETE FROM transactions WHERE partner_reference = ANY($1::text[]) OR partner_reference LIKE $2 RETURNING id",
|
||||
[created.partnerReferences, `LOAD-%-${RUN_ID}-%`]
|
||||
);
|
||||
const deletedMerchants = await pool.query(
|
||||
"DELETE FROM merchants WHERE id = ANY($1::text[]) OR legal_name = $2 RETURNING id",
|
||||
[created.merchantIds, `Load Test Merchant ${RUN_ID}`]
|
||||
);
|
||||
return {
|
||||
transactions_deleted: deletedTransactions.rowCount,
|
||||
merchants_deleted: deletedMerchants.rowCount
|
||||
};
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
function printSummary(startedAt, cleanupResult) {
|
||||
const totalDurationMs = performance.now() - startedAt;
|
||||
const rows = [...metrics.entries()].map(([label, entry]) => ({
|
||||
label,
|
||||
ok: entry.ok,
|
||||
error: entry.error,
|
||||
count: entry.durations.length,
|
||||
p50_ms: Number(percentile(entry.durations, 50).toFixed(2)),
|
||||
p95_ms: Number(percentile(entry.durations, 95).toFixed(2)),
|
||||
max_ms: Number(Math.max(...entry.durations).toFixed(2))
|
||||
}));
|
||||
const totalOk = rows.reduce((sum, row) => sum + row.ok, 0);
|
||||
const totalError = rows.reduce((sum, row) => sum + row.error, 0);
|
||||
const summary = {
|
||||
run_id: RUN_ID,
|
||||
base_url: BASE,
|
||||
config: {
|
||||
callbacks: CALLBACKS,
|
||||
heartbeats: HEARTBEATS,
|
||||
dynamic_qr: DYNAMIC_QR,
|
||||
reads: READS,
|
||||
exports: EXPORTS,
|
||||
concurrency: CONCURRENCY
|
||||
},
|
||||
totals: {
|
||||
ok: totalOk,
|
||||
error: totalError,
|
||||
duration_ms: Number(totalDurationMs.toFixed(2)),
|
||||
approx_throughput_rps: Number((totalOk / (totalDurationMs / 1000)).toFixed(2))
|
||||
},
|
||||
metrics: rows,
|
||||
cleanup: cleanupResult
|
||||
};
|
||||
const serialized = JSON.stringify(summary, null, 2);
|
||||
console.log(serialized);
|
||||
if (process.env.LOAD_REPORT_FILE) {
|
||||
const reportFile = path.resolve(process.cwd(), process.env.LOAD_REPORT_FILE);
|
||||
fs.mkdirSync(path.dirname(reportFile), { recursive: true });
|
||||
fs.writeFileSync(reportFile, `${serialized}\n`, "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const startedAt = performance.now();
|
||||
let cleanupResult = null;
|
||||
try {
|
||||
await waitForHealth();
|
||||
const bundle = await createBundle();
|
||||
|
||||
const callbackIndexes = Array.from({ length: CALLBACKS }, (_item, index) => index + 1);
|
||||
await runPool(callbackIndexes, CONCURRENCY, (index) => createTransaction(bundle, index));
|
||||
await runPool(callbackIndexes, CONCURRENCY, callbackPaid);
|
||||
|
||||
await runPool(Array.from({ length: HEARTBEATS }, (_item, index) => index + 1), CONCURRENCY, (index) =>
|
||||
heartbeat(bundle, index)
|
||||
);
|
||||
await runPool(Array.from({ length: DYNAMIC_QR }, (_item, index) => index + 1), CONCURRENCY, (index) =>
|
||||
dynamicQr(bundle, index)
|
||||
);
|
||||
await runPool(Array.from({ length: READS }, (_item, index) => index + 1), CONCURRENCY, async () => {
|
||||
await reqAdmin("/admin/observability/summary", { label: "observability_summary" });
|
||||
});
|
||||
await runPool(Array.from({ length: EXPORTS }, (_item, index) => index + 1), Math.min(CONCURRENCY, 5), exportAdjustments);
|
||||
|
||||
cleanupResult = await cleanup();
|
||||
printSummary(startedAt, cleanupResult);
|
||||
} catch (error) {
|
||||
cleanupResult = await cleanup().catch((cleanupError) => ({ error: cleanupError.message }));
|
||||
printSummary(startedAt, cleanupResult);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user