Production readiness hardening and ops tooling

scripts/backup-production.mjs

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import "dotenv/config";
+
+const args = process.argv.slice(2);
+
+function getArg(name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+function hasFlag(name) {
+  return args.includes(name);
+}
+
+function usage() {
+  console.log(`Usage:
+  node scripts/backup-production.mjs [--out ./backups] [--include-mosquitto] [--dry-run]
+
+Creates a timestamped PostgreSQL custom-format dump. When --include-mosquitto is set,
+copies Mosquitto passwd/ACL files if readable.
+`);
+}
+
+if (hasFlag("--help") || hasFlag("-h")) {
+  usage();
+  process.exit(0);
+}
+
+const outDir = path.resolve(process.cwd(), getArg("--out") || process.env.BACKUP_DIR || "./backups");
+const includeMosquitto = hasFlag("--include-mosquitto");
+const dryRun = hasFlag("--dry-run");
+const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+const dbName = process.env.PGDATABASE || "qris_soundbox_platform";
+const dbBackup = path.join(outDir, `${dbName}-${timestamp}.dump`);
+const passwdFile = process.env.MOSQUITTO_PASSWD_FILE || "/etc/mosquitto/passwd";
+const aclFile = process.env.MOSQUITTO_ACL_FILE || "/etc/mosquitto/acl";
+
+fs.mkdirSync(outDir, { recursive: true });
+
+const pgDumpArgs = ["-Fc", "-f", dbBackup];
+if (process.env.DATABASE_URL) {
+  pgDumpArgs.push(process.env.DATABASE_URL);
+} else {
+  pgDumpArgs.push("-h", process.env.PGHOST || "127.0.0.1");
+  pgDumpArgs.push("-p", String(process.env.PGPORT || 5432));
+  pgDumpArgs.push("-U", process.env.PGUSER || "postgres");
+  pgDumpArgs.push(dbName);
+}
+
+const plan = {
+  out_dir: outDir,
+  database_backup: dbBackup,
+  include_mosquitto: includeMosquitto,
+  commands: [`pg_dump ${pgDumpArgs.map((arg) => (arg.includes(" ") ? JSON.stringify(arg) : arg)).join(" ")}`]
+};
+
+if (dryRun) {
+  console.log(JSON.stringify({ dry_run: true, ...plan }, null, 2));
+  process.exit(0);
+}
+
+const result = spawnSync("pg_dump", pgDumpArgs, {
+  stdio: "inherit",
+  env: process.env
+});
+if (result.status !== 0) {
+  throw new Error(`pg_dump failed with status ${result.status}`);
+}
+
+const copied = [];
+if (includeMosquitto) {
+  for (const source of [passwdFile, aclFile]) {
+    if (!fs.existsSync(source)) {
+      continue;
+    }
+    const target = path.join(outDir, `${path.basename(source)}-${timestamp}`);
+    fs.copyFileSync(source, target);
+    copied.push({ source, target });
+  }
+}
+
+console.log(JSON.stringify({ ok: true, ...plan, copied }, null, 2));

scripts/check-mqtt-acl.mjs

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+
+const args = process.argv.slice(2);
+
+function getArg(name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+function hasFlag(name) {
+  return args.includes(name);
+}
+
+function usage() {
+  console.log(`Usage:
+  node scripts/check-mqtt-acl.mjs --file /etc/mosquitto/acl
+  node scripts/check-mqtt-acl.mjs --print-template
+
+Required Mosquitto ACL rules:
+  pattern readwrite devices/%u/uplink/#
+  pattern read devices/%u/downlink/#
+  user qris-backend
+  topic read devices/+/uplink/#
+  topic write devices/+/downlink/#
+`);
+}
+
+const requiredLines = [
+  "pattern readwrite devices/%u/uplink/#",
+  "pattern read devices/%u/downlink/#",
+  "user qris-backend",
+  "topic read devices/+/uplink/#",
+  "topic write devices/+/downlink/#"
+];
+
+if (hasFlag("--help") || hasFlag("-h")) {
+  usage();
+  process.exit(0);
+}
+
+if (hasFlag("--print-template")) {
+  console.log(`# QRIS Soundbox Mosquitto ACL
+# Device username must equal platform device_id.
+${requiredLines.join("\n")}
+`);
+  process.exit(0);
+}
+
+const aclFile = getArg("--file") || process.env.MOSQUITTO_ACL_FILE;
+if (!aclFile) {
+  usage();
+  process.exit(1);
+}
+
+const content = fs.readFileSync(aclFile, "utf8");
+const normalized = content
+  .split(/\r?\n/)
+  .map((line) => line.trim())
+  .filter((line) => line && !line.startsWith("#"));
+
+const missing = requiredLines.filter((line) => !normalized.includes(line));
+const result = {
+  file: aclFile,
+  ok: missing.length === 0,
+  missing,
+  required: requiredLines
+};
+
+console.log(JSON.stringify(result, null, 2));
+process.exit(result.ok ? 0 : 1);

scripts/check-production-env.mjs

@@ -0,0 +1,141 @@
+import "dotenv/config";
+import path from "node:path";
+
+const required = [
+  "ADMIN_SESSION_SECRET",
+  "MERCHANT_SESSION_SECRET",
+  "INTEGRATION_WEBHOOK_SECRET",
+  "DATABASE_URL",
+  "MQTT_BROKER_URL",
+  "MQTT_USERNAME",
+  "MQTT_PASSWORD",
+  "MQTT_CLIENT_ID",
+  "EXPORT_STORAGE_DIR"
+];
+
+const insecureDefaults = new Map([
+  ["ADMIN_TOKEN", "admin-dev-token"],
+  ["MERCHANT_TOKEN", "merchant-dev-token"],
+  ["DEVICE_TOKEN", "device-dev-token"],
+  ["MERCHANT_PORTAL_PASSWORD", "merchant"],
+  ["INTEGRATION_WEBHOOK_SECRET", "dev-callback-secret"],
+  ["MQTT_PASSWORD", "change-me"],
+  ["ADMIN_SESSION_SECRET", "change-me-long-random-admin-session-secret"],
+  ["MERCHANT_SESSION_SECRET", "change-me-long-random-merchant-session-secret"]
+]);
+
+const warnings = [];
+const errors = [];
+
+for (const name of required) {
+  if (!process.env[name]) {
+    errors.push(`${name} is required`);
+  }
+}
+
+for (const [name, value] of insecureDefaults.entries()) {
+  if (name === "ADMIN_TOKEN" && process.env.ADMIN_AUTH_ALLOW_LEGACY_TOKEN === "false") {
+    continue;
+  }
+  if (name === "DEVICE_TOKEN" && process.env.DEVICE_AUTH_ALLOW_LEGACY_TOKEN === "false") {
+    continue;
+  }
+  if (name === "MERCHANT_TOKEN" && process.env.MERCHANT_AUTH_ALLOW_LEGACY_TOKEN === "false") {
+    continue;
+  }
+  if (name === "MERCHANT_PORTAL_PASSWORD" && process.env.MERCHANT_DEV_LOGIN_ENABLED === "false") {
+    continue;
+  }
+  if (process.env[name] === value) {
+    errors.push(`${name} still uses the development default`);
+  }
+}
+
+function requireFalse(name) {
+  if (process.env[name] !== "false") {
+    errors.push(`${name} must be false in production`);
+  }
+}
+
+function requireTrue(name, message) {
+  if (process.env[name] !== "true") {
+    errors.push(message || `${name} must be true in production`);
+  }
+}
+
+requireFalse("ADMIN_AUTH_ALLOW_LEGACY_TOKEN");
+requireFalse("DEVICE_AUTH_ALLOW_LEGACY_TOKEN");
+requireFalse("MERCHANT_AUTH_ALLOW_LEGACY_TOKEN");
+
+if (process.env.ADMIN_DEV_LOGIN_ENABLED !== "false") {
+  errors.push("ADMIN_DEV_LOGIN_ENABLED must be false in production");
+}
+
+if (process.env.MERCHANT_DEV_LOGIN_ENABLED !== "false") {
+  errors.push("MERCHANT_DEV_LOGIN_ENABLED must be false in production");
+}
+
+if (process.env.MQTT_PUBLISH_MODE !== "broker") {
+  errors.push("MQTT_PUBLISH_MODE must be broker in production");
+}
+
+if (process.env.MQTT_SUBSCRIBE_ENABLED !== "true") {
+  warnings.push("MQTT_SUBSCRIBE_ENABLED should be true when device uplink observability is required");
+}
+
+requireTrue("SETTLEMENT_ADJUSTMENT_REQUIRE_APPROVAL", "SETTLEMENT_ADJUSTMENT_REQUIRE_APPROVAL must be true for production finance control");
+
+if (process.env.DATABASE_URL && !/^postgres(ql)?:\/\//.test(process.env.DATABASE_URL)) {
+  errors.push("DATABASE_URL must be a postgres connection string");
+}
+
+if (process.env.MQTT_BROKER_URL && !/^mqtts:\/\//.test(process.env.MQTT_BROKER_URL)) {
+  warnings.push("MQTT_BROKER_URL should use mqtts:// for production");
+}
+
+for (const name of ["ADMIN_SESSION_SECRET", "MERCHANT_SESSION_SECRET", "INTEGRATION_WEBHOOK_SECRET", "MQTT_PASSWORD"]) {
+  if ((process.env[name] || "").length < 24) {
+    errors.push(`${name} must be at least 24 characters`);
+  }
+}
+
+if (process.env.LOG_FORMAT !== "json") {
+  warnings.push("LOG_FORMAT should be json in production");
+}
+
+if (process.env.EXPORT_WORKER_ENABLED !== "true") {
+  errors.push("EXPORT_WORKER_ENABLED must be true in production");
+}
+
+if (Number(process.env.EXPORT_RETENTION_DAYS || 0) <= 0) {
+  errors.push("EXPORT_RETENTION_DAYS must be a positive number");
+}
+
+if (process.env.EXPORT_STORAGE_DIR && !path.isAbsolute(process.env.EXPORT_STORAGE_DIR)) {
+  errors.push("EXPORT_STORAGE_DIR must be an absolute path in production");
+}
+
+if (process.env.RATE_LIMIT_ENABLED !== "true") {
+  errors.push("RATE_LIMIT_ENABLED must be true in production");
+}
+
+if (process.env.TRUST_PROXY !== "true") {
+  warnings.push("TRUST_PROXY should be true when running behind a reverse proxy/load balancer");
+}
+
+if (!process.env.JSON_BODY_LIMIT) {
+  errors.push("JSON_BODY_LIMIT is required");
+}
+
+for (const warning of warnings) {
+  console.warn(`WARN ${warning}`);
+}
+
+if (errors.length) {
+  for (const error of errors) {
+    console.error(`ERROR ${error}`);
+  }
+  process.exit(1);
+}
+
+console.log("Production environment preflight passed");

scripts/create-admin-user.mjs

@@ -0,0 +1,138 @@
+#!/usr/bin/env node
+import { randomBytes, scryptSync } from "node:crypto";
+import { Pool } from "pg";
+import "dotenv/config";
+
+const args = process.argv.slice(2);
+const validRoles = new Set(["admin", "finance", "ops", "support", "viewer"]);
+
+function getArg(name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+function hasFlag(name) {
+  return args.includes(name);
+}
+
+function usage() {
+  console.log(`Usage:
+  npm run admin:create-user -- --email <email> --name <name> --role <admin|finance|ops|support|viewer> --password <password> [--inactive] [--rotate-password]
+
+Examples:
+  npm run admin:create-user -- --email finance@example.com --name "Finance Ops" --role finance --password "change-this"
+  npm run admin:create-user -- --email admin@example.com --name "Main Admin" --role admin --password "change-this" --rotate-password
+
+Environment:
+  DATABASE_URL or PGHOST/PGPORT/PGUSER/PGPASSWORD/PGDATABASE
+`);
+}
+
+function validatePasswordPolicy(value) {
+  const failures = [];
+  if (value.length < 14) failures.push("at least 14 characters");
+  if (!/[a-z]/.test(value)) failures.push("a lowercase letter");
+  if (!/[A-Z]/.test(value)) failures.push("an uppercase letter");
+  if (!/[0-9]/.test(value)) failures.push("a number");
+  if (!/[^A-Za-z0-9]/.test(value)) failures.push("a symbol");
+  if (/password|admin|qris|soundbox|change-this/i.test(value)) failures.push("no obvious product/default words");
+  return failures;
+}
+
+const email = (getArg("--email") || "").trim().toLowerCase();
+const name = (getArg("--name") || "").trim();
+const roleName = (getArg("--role") || "").trim().toLowerCase();
+const password = getArg("--password") || "";
+const status = hasFlag("--inactive") ? "inactive" : "active";
+const rotatePassword = hasFlag("--rotate-password");
+
+if (hasFlag("--help") || hasFlag("-h")) {
+  usage();
+  process.exit(0);
+}
+
+const errors = [];
+if (!email || !/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(email)) {
+  errors.push("--email must be a valid email address");
+}
+if (!name) {
+  errors.push("--name is required");
+}
+if (!validRoles.has(roleName)) {
+  errors.push("--role must be one of admin, finance, ops, support, viewer");
+}
+const passwordFailures = validatePasswordPolicy(password);
+if (passwordFailures.length) {
+  errors.push(`--password must include ${passwordFailures.join(", ")}`);
+}
+if (errors.length) {
+  usage();
+  for (const error of errors) {
+    console.error(`ERROR ${error}`);
+  }
+  process.exit(1);
+}
+
+function makePool() {
+  if (process.env.DATABASE_URL) {
+    return new Pool({ connectionString: process.env.DATABASE_URL });
+  }
+  return new Pool({
+    host: process.env.PGHOST || "127.0.0.1",
+    port: Number(process.env.PGPORT || 5432),
+    user: process.env.PGUSER || "postgres",
+    password: process.env.PGPASSWORD || "postgres",
+    database: process.env.PGDATABASE || "qris_soundbox_platform"
+  });
+}
+
+function hashPassword(value) {
+  const salt = randomBytes(16).toString("hex");
+  const keyLength = 64;
+  const derived = scryptSync(value, salt, keyLength).toString("hex");
+  return `scrypt$${salt}$${keyLength}$${derived}`;
+}
+
+function userIdFromEmail(value) {
+  return `user_${value.replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 64)}`;
+}
+
+const pool = makePool();
+
+try {
+  const role = await pool.query("SELECT * FROM roles WHERE name = $1", [roleName]);
+  if (!role.rows.length) {
+    throw new Error(`role '${roleName}' not found. Run npm run db:migrate so schema/seed roles are initialized.`);
+  }
+
+  const existing = await pool.query("SELECT id, email FROM users WHERE LOWER(email) = LOWER($1)", [email]);
+  const passwordHash = hashPassword(password);
+  const now = new Date().toISOString();
+
+  if (existing.rows.length) {
+    const updatePasswordSql = rotatePassword ? ", password_hash = $6" : "";
+    const params = rotatePassword
+      ? [name, role.rows[0].id, status, now, email, passwordHash]
+      : [name, role.rows[0].id, status, now, email];
+    await pool.query(
+      `UPDATE users
+          SET name = $1,
+              role_id = $2,
+              status = $3,
+              created_at = COALESCE(created_at, $4)
+              ${updatePasswordSql}
+        WHERE LOWER(email) = LOWER($5)`,
+      params
+    );
+    console.log(`Updated admin user ${email} role=${roleName} status=${status} password_rotated=${rotatePassword}`);
+  } else {
+    await pool.query(
+      `INSERT INTO users (id, name, email, password_hash, role_id, status, created_at)
+       VALUES ($1,$2,$3,$4,$5,$6,$7)`,
+      [userIdFromEmail(email), name, email, passwordHash, role.rows[0].id, status, now]
+    );
+    console.log(`Created admin user ${email} role=${roleName} status=${status}`);
+  }
+} finally {
+  await pool.end();
+}

scripts/create-merchant-user.mjs

@@ -0,0 +1,143 @@
+#!/usr/bin/env node
+import { randomBytes, randomUUID, scryptSync } from "node:crypto";
+import { Pool } from "pg";
+import "dotenv/config";
+
+const args = process.argv.slice(2);
+const validRoles = new Set(["owner", "finance", "ops", "viewer"]);
+
+function getArg(name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+function hasFlag(name) {
+  return args.includes(name);
+}
+
+function usage() {
+  console.log(`Usage:
+  npm run merchant:create-user -- --merchant <merchant-id-or-code> --email <email> --name <name> --role <owner|finance|ops|viewer> --password <password> [--inactive] [--rotate-password]
+
+Examples:
+  npm run merchant:create-user -- --merchant m_abc123 --email owner@merchant.com --name "Merchant Owner" --role owner --password "change-this"
+  npm run merchant:create-user -- --merchant <merchant-id> --email finance@merchant.com --name "Merchant Finance" --role finance --password "change-this" --rotate-password
+
+Environment:
+  DATABASE_URL or PGHOST/PGPORT/PGUSER/PGPASSWORD/PGDATABASE
+`);
+}
+
+function validatePasswordPolicy(value) {
+  const failures = [];
+  if (value.length < 14) failures.push("at least 14 characters");
+  if (!/[a-z]/.test(value)) failures.push("a lowercase letter");
+  if (!/[A-Z]/.test(value)) failures.push("an uppercase letter");
+  if (!/[0-9]/.test(value)) failures.push("a number");
+  if (!/[^A-Za-z0-9]/.test(value)) failures.push("a symbol");
+  if (/password|merchant|qris|soundbox|change-this/i.test(value)) failures.push("no obvious product/default words");
+  return failures;
+}
+
+const merchantRef = (getArg("--merchant") || "").trim();
+const email = (getArg("--email") || "").trim().toLowerCase();
+const name = (getArg("--name") || "").trim();
+const roleName = (getArg("--role") || "owner").trim().toLowerCase();
+const password = getArg("--password") || "";
+const status = hasFlag("--inactive") ? "inactive" : "active";
+const rotatePassword = hasFlag("--rotate-password");
+
+if (hasFlag("--help") || hasFlag("-h")) {
+  usage();
+  process.exit(0);
+}
+
+const errors = [];
+if (!merchantRef) {
+  errors.push("--merchant is required");
+}
+if (!email || !/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(email)) {
+  errors.push("--email must be a valid email address");
+}
+if (!name) {
+  errors.push("--name is required");
+}
+if (!validRoles.has(roleName)) {
+  errors.push("--role must be one of owner, finance, ops, viewer");
+}
+const passwordFailures = validatePasswordPolicy(password);
+if (passwordFailures.length) {
+  errors.push(`--password must include ${passwordFailures.join(", ")}`);
+}
+if (errors.length) {
+  usage();
+  for (const error of errors) {
+    console.error(`ERROR ${error}`);
+  }
+  process.exit(1);
+}
+
+function makePool() {
+  if (process.env.DATABASE_URL) {
+    return new Pool({ connectionString: process.env.DATABASE_URL });
+  }
+  return new Pool({
+    host: process.env.PGHOST || "127.0.0.1",
+    port: Number(process.env.PGPORT || 5432),
+    user: process.env.PGUSER || "postgres",
+    password: process.env.PGPASSWORD || "postgres",
+    database: process.env.PGDATABASE || "qris_soundbox_platform"
+  });
+}
+
+function hashPassword(value) {
+  const salt = randomBytes(16).toString("hex");
+  const keyLength = 64;
+  const derived = scryptSync(value, salt, keyLength).toString("hex");
+  return `scrypt$${salt}$${keyLength}$${derived}`;
+}
+
+const pool = makePool();
+
+try {
+  const merchant = await pool.query(
+    "SELECT * FROM merchants WHERE id = $1 OR merchant_code = $1 LIMIT 1",
+    [merchantRef]
+  );
+  if (!merchant.rows.length) {
+    throw new Error(`merchant '${merchantRef}' not found`);
+  }
+
+  const merchantId = merchant.rows[0].id;
+  const existing = await pool.query("SELECT id, email FROM merchant_users WHERE LOWER(email) = LOWER($1)", [email]);
+  const passwordHash = hashPassword(password);
+  const now = new Date().toISOString();
+
+  if (existing.rows.length) {
+    const updatePasswordSql = rotatePassword ? ", password_hash = $7" : "";
+    const params = rotatePassword
+      ? [merchantId, name, roleName, status, now, email, passwordHash]
+      : [merchantId, name, roleName, status, now, email];
+    await pool.query(
+      `UPDATE merchant_users
+          SET merchant_id = $1,
+              name = $2,
+              role_name = $3,
+              status = $4,
+              updated_at = $5
+              ${updatePasswordSql}
+        WHERE LOWER(email) = LOWER($6)`,
+      params
+    );
+    console.log(`Updated merchant user ${email} merchant=${merchantId} role=${roleName} status=${status} password_rotated=${rotatePassword}`);
+  } else {
+    await pool.query(
+      `INSERT INTO merchant_users (id, merchant_id, name, email, password_hash, role_name, status, created_at, updated_at)
+       VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
+      [`merchant_user_${randomUUID()}`, merchantId, name, email, passwordHash, roleName, status, now, now]
+    );
+    console.log(`Created merchant user ${email} merchant=${merchantId} role=${roleName} status=${status}`);
+  }
+} finally {
+  await pool.end();
+}

scripts/load-test.mjs

@@ -0,0 +1,413 @@
+#!/usr/bin/env node
+import { createHmac } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { Pool } from "pg";
+import "dotenv/config";
+
+const PORT = process.env.PORT || "3120";
+const BASE = process.env.BASE_URL || `http://127.0.0.1:${PORT}`;
+const ADMIN_TOKEN = process.env.ADMIN_TOKEN || "admin-dev-token";
+const DEVICE_TOKEN = process.env.DEVICE_TOKEN || "device-dev-token";
+const SECRET = process.env.INTEGRATION_WEBHOOK_SECRET || "dev-callback-secret";
+
+const RUN_ID = process.env.LOAD_RUN_ID || `load-${Date.now()}`;
+const CALLBACKS = Number(process.env.LOAD_CALLBACKS || 30);
+const HEARTBEATS = Number(process.env.LOAD_HEARTBEATS || 60);
+const DYNAMIC_QR = Number(process.env.LOAD_DYNAMIC_QR || 30);
+const READS = Number(process.env.LOAD_READS || 30);
+const EXPORTS = Number(process.env.LOAD_EXPORTS || 0);
+const CONCURRENCY = Number(process.env.LOAD_CONCURRENCY || 10);
+
+const created = {
+  merchantIds: [],
+  partnerReferences: []
+};
+
+const metrics = new Map();
+
+function percentile(values, p) {
+  if (!values.length) {
+    return 0;
+  }
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
+  return sorted[index];
+}
+
+function record(label, durationMs, ok) {
+  const entry = metrics.get(label) || { durations: [], ok: 0, error: 0 };
+  entry.durations.push(durationMs);
+  if (ok) {
+    entry.ok += 1;
+  } else {
+    entry.error += 1;
+  }
+  metrics.set(label, entry);
+}
+
+async function req(path, options = {}) {
+  const startedAt = performance.now();
+  const label = options.label || `${options.method || "GET"} ${path}`;
+  try {
+    const response = await fetch(`${BASE}${path}`, {
+      method: options.method || "GET",
+      headers: {
+        "Content-Type": "application/json",
+        ...(options.headers || {})
+      },
+      body: Object.prototype.hasOwnProperty.call(options, "body") ? JSON.stringify(options.body) : undefined
+    });
+    const text = await response.text();
+    let body = null;
+    try {
+      body = text ? JSON.parse(text) : null;
+    } catch {
+      body = text;
+    }
+    const ok = response.ok;
+    record(label, performance.now() - startedAt, ok);
+    if (!ok) {
+      throw new Error(`${label} failed status=${response.status} body=${typeof body === "string" ? body.slice(0, 120) : JSON.stringify(body).slice(0, 120)}`);
+    }
+    return body?.data !== undefined ? body.data : body;
+  } catch (error) {
+    record(label, performance.now() - startedAt, false);
+    throw error;
+  }
+}
+
+function reqAdmin(path, options = {}) {
+  return req(path, {
+    ...options,
+    headers: {
+      ...(options.headers || {}),
+      Authorization: `Bearer ${ADMIN_TOKEN}`
+    }
+  });
+}
+
+function reqDevice(path, options = {}) {
+  return req(path, {
+    ...options,
+    headers: {
+      ...(options.headers || {}),
+      Authorization: `Bearer ${DEVICE_TOKEN}`
+    }
+  });
+}
+
+async function runPool(items, concurrency, worker) {
+  const queue = [...items];
+  const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+    while (queue.length) {
+      const item = queue.shift();
+      await worker(item);
+    }
+  });
+  await Promise.all(workers);
+}
+
+async function waitForHealth() {
+  for (let i = 0; i < 100; i += 1) {
+    try {
+      await req("/health", { label: "health_check" });
+      return;
+    } catch {
+      await new Promise((resolve) => setTimeout(resolve, 200));
+    }
+  }
+  throw new Error("health check timeout");
+}
+
+async function createBundle() {
+  const merchant = await reqAdmin("/admin/merchants", {
+    method: "POST",
+    body: {
+      legal_name: `Load Test Merchant ${RUN_ID}`,
+      brand_name: `LOAD-${RUN_ID}`,
+      settlement_account_reference: `bank:${RUN_ID}`,
+      settlement_account_type: "merchant_bank_account",
+      payout_mode: "merchant_direct"
+    },
+    label: "setup_create_merchant"
+  });
+  created.merchantIds.push(merchant.id);
+
+  const outlet = await reqAdmin(`/admin/merchants/${merchant.id}/outlets`, {
+    method: "POST",
+    body: { name: `Load Outlet ${RUN_ID}` },
+    label: "setup_create_outlet"
+  });
+  const terminal = await reqAdmin(`/admin/outlets/${outlet.id}/terminals`, {
+    method: "POST",
+    body: { terminal_code: `LOAD-TERM-${RUN_ID}`, qr_mode: "static" },
+    label: "setup_create_terminal"
+  });
+  const device = await reqAdmin("/admin/devices", {
+    method: "POST",
+    body: {
+      device_code: `LOAD-DEV-${RUN_ID}`,
+      vendor: "load",
+      model: "static",
+      communication_mode: "mqtt",
+      status: "active"
+    },
+    label: "setup_create_device"
+  });
+  await reqAdmin(`/admin/devices/${device.id}/bind`, {
+    method: "POST",
+    body: {
+      merchant_id: merchant.id,
+      outlet_id: outlet.id,
+      terminal_id: terminal.id
+    },
+    label: "setup_bind_device"
+  });
+
+  const dynamicOutlet = await reqAdmin(`/admin/merchants/${merchant.id}/outlets`, {
+    method: "POST",
+    body: { name: `Load Dynamic Outlet ${RUN_ID}` },
+    label: "setup_create_dynamic_outlet"
+  });
+  const dynamicTerminal = await reqAdmin(`/admin/outlets/${dynamicOutlet.id}/terminals`, {
+    method: "POST",
+    body: { terminal_code: `LOAD-DYN-${RUN_ID}`, qr_mode: "dynamic_api" },
+    label: "setup_create_dynamic_terminal"
+  });
+  const dynamicDevice = await reqAdmin("/admin/devices", {
+    method: "POST",
+    body: {
+      device_code: `LOAD-DYN-DEV-${RUN_ID}`,
+      vendor: "load",
+      model: "dynamic-api",
+      communication_mode: "api",
+      capability_profile_json: {
+        dynamic_qr: { api_direct: true, mqtt: false },
+        flows: ["dynamic_qr:api_direct", "static_payment_notification"]
+      },
+      status: "active"
+    },
+    label: "setup_create_dynamic_device"
+  });
+  await reqAdmin(`/admin/devices/${dynamicDevice.id}/bind`, {
+    method: "POST",
+    body: {
+      merchant_id: merchant.id,
+      outlet_id: dynamicOutlet.id,
+      terminal_id: dynamicTerminal.id
+    },
+    label: "setup_bind_dynamic_device"
+  });
+
+  return {
+    merchant,
+    outlet,
+    terminal,
+    device,
+    dynamicOutlet,
+    dynamicTerminal,
+    dynamicDevice
+  };
+}
+
+async function createTransaction(bundle, index) {
+  const partnerReference = `LOAD-PR-${RUN_ID}-${index}`;
+  created.partnerReferences.push(partnerReference);
+  return reqAdmin("/admin/transactions", {
+    method: "POST",
+    body: {
+      partner_reference: partnerReference,
+      merchant_id: bundle.merchant.id,
+      outlet_id: bundle.outlet.id,
+      terminal_id: bundle.terminal.id,
+      device_id: bundle.device.id,
+      amount: 10000 + index,
+      currency: "IDR",
+      qr_mode: "static",
+      initiation_mode: "static",
+      status: "initiated"
+    },
+    label: "transaction_create"
+  });
+}
+
+async function callbackPaid(index) {
+  const callback = {
+    partner_reference: `LOAD-PR-${RUN_ID}-${index}`,
+    partner_txn_id: `LOAD-PTX-${RUN_ID}-${index}`,
+    amount: 10000 + index,
+    currency: "IDR",
+    payment_status: "paid",
+    status: "paid",
+    paid_at: new Date().toISOString()
+  };
+  const signature = createHmac("sha256", SECRET).update(JSON.stringify(callback)).digest("hex");
+  return req("/integrations/qris/callback", {
+    method: "POST",
+    headers: { "X-Partner-Signature": signature },
+    body: { ...callback, signature },
+    label: "qris_callback_paid"
+  });
+}
+
+async function heartbeat(bundle, index) {
+  return reqDevice("/device/heartbeat", {
+    method: "POST",
+    body: {
+      device_id: bundle.device.id,
+      timestamp: new Date().toISOString(),
+      firmware_version: "load-1.0.0",
+      network_strength: 70 + (index % 20),
+      battery_level: 60 + (index % 30),
+      state: "load-test"
+    },
+    label: "device_heartbeat"
+  });
+}
+
+async function dynamicQr(bundle, index) {
+  const requestId = `LOAD-DYN-${RUN_ID}-${index}`;
+  return reqDevice("/device/transactions/dynamic-qr", {
+    method: "POST",
+    headers: { "Idempotency-Key": requestId },
+    body: {
+      device_id: bundle.dynamicDevice.id,
+      terminal_id: bundle.dynamicTerminal.id,
+      amount: 15000 + index,
+      currency: "IDR",
+      request_id: requestId,
+      expires_in_seconds: 300
+    },
+    label: "dynamic_qr_create"
+  });
+}
+
+async function exportAdjustments(index) {
+  const job = await reqAdmin("/admin/exports/settlement-adjustments", {
+    method: "POST",
+    body: { limit: 500 },
+    label: "export_adjustments_create"
+  });
+  let current = job;
+  for (let attempt = 0; attempt < 30 && !["completed", "failed"].includes(current.status); attempt += 1) {
+    await new Promise((resolve) => setTimeout(resolve, 500));
+    current = await reqAdmin(`/admin/exports/${job.id}`, {
+      label: "export_adjustments_poll"
+    });
+  }
+  if (current.status !== "completed") {
+    throw new Error(`export job ${index} ended with status ${current.status}`);
+  }
+  await reqAdmin(`/admin/exports/${job.id}/download`, {
+    label: "export_adjustments_download"
+  });
+}
+
+async function cleanup() {
+  const pool = new Pool(
+    process.env.DATABASE_URL
+      ? { connectionString: process.env.DATABASE_URL }
+      : {
+          host: process.env.PGHOST || "127.0.0.1",
+          port: Number(process.env.PGPORT || 5432),
+          user: process.env.PGUSER || "postgres",
+          password: process.env.PGPASSWORD || "postgres",
+          database: process.env.PGDATABASE || "qris_soundbox_platform"
+        }
+  );
+  try {
+    const deletedTransactions = await pool.query(
+      "DELETE FROM transactions WHERE partner_reference = ANY($1::text[]) OR partner_reference LIKE $2 RETURNING id",
+      [created.partnerReferences, `LOAD-%-${RUN_ID}-%`]
+    );
+    const deletedMerchants = await pool.query(
+      "DELETE FROM merchants WHERE id = ANY($1::text[]) OR legal_name = $2 RETURNING id",
+      [created.merchantIds, `Load Test Merchant ${RUN_ID}`]
+    );
+    return {
+      transactions_deleted: deletedTransactions.rowCount,
+      merchants_deleted: deletedMerchants.rowCount
+    };
+  } finally {
+    await pool.end();
+  }
+}
+
+function printSummary(startedAt, cleanupResult) {
+  const totalDurationMs = performance.now() - startedAt;
+  const rows = [...metrics.entries()].map(([label, entry]) => ({
+    label,
+    ok: entry.ok,
+    error: entry.error,
+    count: entry.durations.length,
+    p50_ms: Number(percentile(entry.durations, 50).toFixed(2)),
+    p95_ms: Number(percentile(entry.durations, 95).toFixed(2)),
+    max_ms: Number(Math.max(...entry.durations).toFixed(2))
+  }));
+  const totalOk = rows.reduce((sum, row) => sum + row.ok, 0);
+  const totalError = rows.reduce((sum, row) => sum + row.error, 0);
+  const summary = {
+      run_id: RUN_ID,
+      base_url: BASE,
+      config: {
+        callbacks: CALLBACKS,
+        heartbeats: HEARTBEATS,
+        dynamic_qr: DYNAMIC_QR,
+        reads: READS,
+        exports: EXPORTS,
+        concurrency: CONCURRENCY
+      },
+      totals: {
+        ok: totalOk,
+        error: totalError,
+        duration_ms: Number(totalDurationMs.toFixed(2)),
+        approx_throughput_rps: Number((totalOk / (totalDurationMs / 1000)).toFixed(2))
+      },
+      metrics: rows,
+      cleanup: cleanupResult
+    };
+  const serialized = JSON.stringify(summary, null, 2);
+  console.log(serialized);
+  if (process.env.LOAD_REPORT_FILE) {
+    const reportFile = path.resolve(process.cwd(), process.env.LOAD_REPORT_FILE);
+    fs.mkdirSync(path.dirname(reportFile), { recursive: true });
+    fs.writeFileSync(reportFile, `${serialized}\n`, "utf8");
+  }
+}
+
+async function main() {
+  const startedAt = performance.now();
+  let cleanupResult = null;
+  try {
+    await waitForHealth();
+    const bundle = await createBundle();
+
+    const callbackIndexes = Array.from({ length: CALLBACKS }, (_item, index) => index + 1);
+    await runPool(callbackIndexes, CONCURRENCY, (index) => createTransaction(bundle, index));
+    await runPool(callbackIndexes, CONCURRENCY, callbackPaid);
+
+    await runPool(Array.from({ length: HEARTBEATS }, (_item, index) => index + 1), CONCURRENCY, (index) =>
+      heartbeat(bundle, index)
+    );
+    await runPool(Array.from({ length: DYNAMIC_QR }, (_item, index) => index + 1), CONCURRENCY, (index) =>
+      dynamicQr(bundle, index)
+    );
+    await runPool(Array.from({ length: READS }, (_item, index) => index + 1), CONCURRENCY, async () => {
+      await reqAdmin("/admin/observability/summary", { label: "observability_summary" });
+    });
+    await runPool(Array.from({ length: EXPORTS }, (_item, index) => index + 1), Math.min(CONCURRENCY, 5), exportAdjustments);
+
+    cleanupResult = await cleanup();
+    printSummary(startedAt, cleanupResult);
+  } catch (error) {
+    cleanupResult = await cleanup().catch((cleanupError) => ({ error: cleanupError.message }));
+    printSummary(startedAt, cleanupResult);
+    throw error;
+  }
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : error);
+  process.exit(1);
+});

scripts/migrate.mts

@@ -0,0 +1,112 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { closePool, ensureSchema, getPool } from "../src/shared/db/pool";
+
+type MigrationContext = {
+  ensureSchema: typeof ensureSchema;
+};
+
+type JsMigrationModule = {
+  up?: (context: MigrationContext) => Promise<void>;
+};
+
+const migrationsDir = path.resolve(process.cwd(), "migrations");
+
+async function ensureMigrationTable() {
+  await getPool().query(`
+    CREATE TABLE IF NOT EXISTS schema_migrations (
+      id TEXT PRIMARY KEY,
+      filename TEXT NOT NULL,
+      checksum TEXT,
+      applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    )
+  `);
+}
+
+async function listMigrationFiles() {
+  const entries = await fs.readdir(migrationsDir, { withFileTypes: true });
+  return entries
+    .filter((entry) => entry.isFile())
+    .map((entry) => entry.name)
+    .filter((name) => /^\d+_.+\.(sql|mjs|mts)$/.test(name))
+    .sort((a, b) => a.localeCompare(b));
+}
+
+async function getAppliedMigrationIds() {
+  const { rows } = await getPool().query("SELECT id FROM schema_migrations ORDER BY id ASC");
+  return new Set(rows.map((row) => String(row.id)));
+}
+
+function migrationId(filename: string) {
+  return filename.split("_")[0];
+}
+
+async function runSqlMigration(filePath: string) {
+  const sql = await fs.readFile(filePath, "utf8");
+  if (!sql.trim()) {
+    return;
+  }
+  await getPool().query(sql);
+}
+
+async function runJsMigration(filePath: string) {
+  const moduleUrl = pathToFileURL(filePath).href;
+  const migration = (await import(moduleUrl)) as JsMigrationModule;
+  if (typeof migration.up !== "function") {
+    throw new Error(`${path.basename(filePath)} must export async function up(context)`);
+  }
+  await migration.up({ ensureSchema });
+}
+
+async function recordMigration(id: string, filename: string) {
+  await getPool().query(
+    `INSERT INTO schema_migrations (id, filename, applied_at)
+     VALUES ($1,$2,NOW())
+     ON CONFLICT (id) DO NOTHING`,
+    [id, filename]
+  );
+}
+
+async function main() {
+  await ensureMigrationTable();
+  const lock = await getPool().query("SELECT pg_try_advisory_lock($1) AS locked", [7642001]);
+  if (!lock.rows[0]?.locked) {
+    throw new Error("another migration process is already running");
+  }
+
+  try {
+    const files = await listMigrationFiles();
+    const applied = await getAppliedMigrationIds();
+    let appliedCount = 0;
+
+    for (const filename of files) {
+      const id = migrationId(filename);
+      if (applied.has(id)) {
+        console.log(`skip ${filename}`);
+        continue;
+      }
+
+      const filePath = path.join(migrationsDir, filename);
+      console.log(`apply ${filename}`);
+      if (filename.endsWith(".sql")) {
+        await runSqlMigration(filePath);
+      } else {
+        await runJsMigration(filePath);
+      }
+      await recordMigration(id, filename);
+      appliedCount += 1;
+    }
+
+    console.log(`migration complete applied=${appliedCount}`);
+  } finally {
+    await getPool().query("SELECT pg_advisory_unlock($1)", [7642001]);
+    await closePool();
+  }
+}
+
+main().catch(async (error) => {
+  console.error(error instanceof Error ? error.message : error);
+  await closePool();
+  process.exit(1);
+});

scripts/provision-mqtt-device.mjs

@@ -0,0 +1,108 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import "dotenv/config";
+
+const args = process.argv.slice(2);
+
+function getArg(name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+function hasFlag(name) {
+  return args.includes(name);
+}
+
+function usage() {
+  console.log(`Usage:
+  node scripts/provision-mqtt-device.mjs --device-id <device-id> [--base-url http://127.0.0.1:3000] [--apply-local]
+
+Options:
+  --device-id     Device UUID from the platform.
+  --base-url      Running backend URL. Default: BASE_URL env or http://127.0.0.1:3000.
+  --apply-local   Run mosquitto_passwd locally. Use only on the broker host.
+
+Environment:
+  ADMIN_TOKEN     Admin bearer token. Default: admin-dev-token.
+  MOSQUITTO_PASSWD_FILE  Default: /etc/mosquitto/passwd.
+  MOSQUITTO_ACL_FILE     Default: /etc/mosquitto/acl.
+`);
+}
+
+const deviceId = getArg("--device-id");
+const baseUrl = getArg("--base-url") || process.env.BASE_URL || "http://127.0.0.1:3000";
+const adminToken = process.env.ADMIN_TOKEN || "admin-dev-token";
+const passwdFile = process.env.MOSQUITTO_PASSWD_FILE || "/etc/mosquitto/passwd";
+const aclFile = process.env.MOSQUITTO_ACL_FILE || "/etc/mosquitto/acl";
+const applyLocal = hasFlag("--apply-local");
+
+if (!deviceId || hasFlag("--help") || hasFlag("-h")) {
+  usage();
+  process.exit(deviceId ? 0 : 1);
+}
+
+function shellQuote(value) {
+  return `'${String(value).replace(/'/g, "'\\''")}'`;
+}
+
+async function rotateCredential() {
+  const response = await fetch(`${baseUrl}/admin/devices/${deviceId}/credentials/rotate`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${adminToken}`,
+      "Content-Type": "application/json"
+    },
+    body: "{}"
+  });
+
+  const payload = await response.json().catch(() => ({}));
+  if (!response.ok) {
+    throw new Error(payload?.message || payload?.code || `rotate failed with status ${response.status}`);
+  }
+
+  return payload.data;
+}
+
+function applyMosquittoPassword(username, password) {
+  const result = spawnSync("sudo", ["mosquitto_passwd", "-b", passwdFile, username, password], {
+    stdio: "inherit"
+  });
+  if (result.status !== 0) {
+    throw new Error(`mosquitto_passwd failed with status ${result.status}`);
+  }
+}
+
+const data = await rotateCredential();
+const username = data.credential.mqtt_username;
+const password = data.credential.mqtt_password;
+
+if (applyLocal) {
+  applyMosquittoPassword(username, password);
+}
+
+console.log(
+  JSON.stringify(
+    {
+      device_id: data.device.id,
+      mqtt_username: username,
+      mqtt_password: password,
+      one_time_secret: true,
+      applied_local: applyLocal,
+      mosquitto_commands: [
+        `sudo mosquitto_passwd -b ${shellQuote(passwdFile)} ${shellQuote(username)} ${shellQuote(password)}`,
+        `node scripts/check-mqtt-acl.mjs --file ${shellQuote(aclFile)} || node scripts/check-mqtt-acl.mjs --print-template`,
+        "sudo chown root:mosquitto /etc/mosquitto/passwd",
+        "sudo chmod 640 /etc/mosquitto/passwd",
+        "sudo systemctl reload mosquitto"
+      ],
+      topic_scope: [
+        `device publish/readwrite: devices/${username}/uplink/#`,
+        `device subscribe/read: devices/${username}/downlink/#`,
+        "backend read: devices/+/uplink/#",
+        "backend write: devices/+/downlink/#"
+      ]
+    },
+    null,
+    2
+  )
+);

scripts/restore-drill-validate.mjs

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { Pool } from "pg";
+import "dotenv/config";
+
+const baseUrl = process.env.BASE_URL || `http://127.0.0.1:${process.env.PORT || 3000}`;
+const adminToken = process.env.ADMIN_TOKEN || "admin-dev-token";
+const runMigrate = process.env.RESTORE_DRILL_RUN_MIGRATE !== "false";
+
+async function httpCheck(path, headers = {}) {
+  const response = await fetch(`${baseUrl}${path}`, { headers });
+  const text = await response.text();
+  let body = null;
+  try {
+    body = text ? JSON.parse(text) : null;
+  } catch {
+    body = text;
+  }
+  if (!response.ok) {
+    throw new Error(`${path} failed status=${response.status} body=${text.slice(0, 200)}`);
+  }
+  return body?.data !== undefined ? body.data : body;
+}
+
+async function dbCheck() {
+  const pool = new Pool(
+    process.env.DATABASE_URL
+      ? { connectionString: process.env.DATABASE_URL }
+      : {
+          host: process.env.PGHOST || "127.0.0.1",
+          port: Number(process.env.PGPORT || 5432),
+          user: process.env.PGUSER || "postgres",
+          password: process.env.PGPASSWORD || "postgres",
+          database: process.env.PGDATABASE || "qris_soundbox_platform"
+        }
+  );
+  try {
+    const tables = await pool.query(
+      "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name IN ('merchants','devices','transactions','export_jobs','schema_migrations')"
+    );
+    return tables.rows.map((row) => row.table_name).sort();
+  } finally {
+    await pool.end();
+  }
+}
+
+if (runMigrate) {
+  const result = spawnSync("npm", ["run", "db:migrate"], { stdio: "inherit", env: process.env });
+  if (result.status !== 0) {
+    throw new Error(`db:migrate failed with status ${result.status}`);
+  }
+}
+
+const [health, adminHealth, tables] = await Promise.all([
+  httpCheck("/health"),
+  httpCheck("/admin/health/deep", { Authorization: `Bearer ${adminToken}` }),
+  dbCheck()
+]);
+
+console.log(
+  JSON.stringify(
+    {
+      ok: true,
+      base_url: baseUrl,
+      health,
+      admin_health_status: adminHealth.status,
+      tables
+    },
+    null,
+    2
+  )
+);

scripts/restore-plan.mjs

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import "dotenv/config";
+
+const args = process.argv.slice(2);
+
+function getArg(name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+function hasFlag(name) {
+  return args.includes(name);
+}
+
+function usage() {
+  console.log(`Usage:
+  node scripts/restore-plan.mjs --backup ./backups/qris.dump [--execute]
+
+Default mode prints the restore command only. --execute runs pg_restore against the
+configured PGDATABASE/DATABASE_URL target, so use it only on a prepared restore DB.
+`);
+}
+
+if (hasFlag("--help") || hasFlag("-h")) {
+  usage();
+  process.exit(0);
+}
+
+const backup = getArg("--backup");
+if (!backup) {
+  usage();
+  process.exit(1);
+}
+
+const backupPath = path.resolve(process.cwd(), backup);
+if (!fs.existsSync(backupPath)) {
+  throw new Error(`backup not found: ${backupPath}`);
+}
+
+const dbName = process.env.PGDATABASE || "qris_soundbox_platform";
+const pgRestoreArgs = ["--clean", "--if-exists", "--no-owner"];
+if (process.env.DATABASE_URL) {
+  pgRestoreArgs.push("-d", process.env.DATABASE_URL);
+} else {
+  pgRestoreArgs.push("-h", process.env.PGHOST || "127.0.0.1");
+  pgRestoreArgs.push("-p", String(process.env.PGPORT || 5432));
+  pgRestoreArgs.push("-U", process.env.PGUSER || "postgres");
+  pgRestoreArgs.push("-d", dbName);
+}
+pgRestoreArgs.push(backupPath);
+
+const command = `pg_restore ${pgRestoreArgs.map((arg) => (arg.includes(" ") ? JSON.stringify(arg) : arg)).join(" ")}`;
+
+if (!hasFlag("--execute")) {
+  console.log(JSON.stringify({ execute: false, command, backup: backupPath }, null, 2));
+  process.exit(0);
+}
+
+const result = spawnSync("pg_restore", pgRestoreArgs, {
+  stdio: "inherit",
+  env: process.env
+});
+if (result.status !== 0) {
+  throw new Error(`pg_restore failed with status ${result.status}`);
+}
+
+console.log(JSON.stringify({ ok: true, backup: backupPath }, null, 2));

scripts/run-staging-load-report.mjs

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+
+const reportDir = path.resolve(process.cwd(), process.env.LOAD_REPORT_DIR || "./reports");
+const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+const reportFile = path.join(reportDir, `load-staging-${timestamp}.json`);
+
+fs.mkdirSync(reportDir, { recursive: true });
+
+const env = {
+  ...process.env,
+  LOAD_CALLBACKS: process.env.LOAD_CALLBACKS || "500",
+  LOAD_HEARTBEATS: process.env.LOAD_HEARTBEATS || "1000",
+  LOAD_DYNAMIC_QR: process.env.LOAD_DYNAMIC_QR || "500",
+  LOAD_READS: process.env.LOAD_READS || "200",
+  LOAD_EXPORTS: process.env.LOAD_EXPORTS || "10",
+  LOAD_CONCURRENCY: process.env.LOAD_CONCURRENCY || "30",
+  LOAD_REPORT_FILE: reportFile
+};
+
+const result = spawnSync("node", ["scripts/load-test.mjs"], {
+  stdio: "inherit",
+  env
+});
+
+if (result.status !== 0) {
+  throw new Error(`staging load report failed with status ${result.status}`);
+}
+
+console.log(JSON.stringify({ ok: true, report_file: reportFile }, null, 2));

scripts/smoke-mqtt-acl.mjs

@@ -0,0 +1,128 @@
+#!/usr/bin/env node
+import mqtt from "mqtt";
+import "dotenv/config";
+
+const brokerUrl = process.env.MQTT_BROKER_URL;
+const deviceA = {
+  username: process.env.MQTT_TEST_DEVICE_A_USERNAME,
+  password: process.env.MQTT_TEST_DEVICE_A_PASSWORD
+};
+const deviceB = {
+  username: process.env.MQTT_TEST_DEVICE_B_USERNAME,
+  password: process.env.MQTT_TEST_DEVICE_B_PASSWORD || process.env.MQTT_TEST_DEVICE_A_PASSWORD
+};
+const timeoutMs = Number(process.env.MQTT_ACL_SMOKE_TIMEOUT_MS || 8000);
+
+function usage() {
+  console.log(`Usage:
+  MQTT_BROKER_URL=mqtts://broker:8883 \\
+  MQTT_TEST_DEVICE_A_USERNAME=<device-a-id> \\
+  MQTT_TEST_DEVICE_A_PASSWORD=<secret> \\
+  MQTT_TEST_DEVICE_B_USERNAME=<device-b-id> \\
+  npm run smoke:mqtt-acl
+
+This validates Mosquitto ACL behavior from a device credential perspective:
+  - device A can subscribe/read devices/A/downlink/#
+  - device A can publish/write devices/A/uplink/#
+  - device A is denied subscribe/read to devices/B/downlink/#
+`);
+}
+
+if (!brokerUrl || !deviceA.username || !deviceA.password || !deviceB.username) {
+  usage();
+  process.exit(1);
+}
+
+function connectDevice(device) {
+  const client = mqtt.connect(brokerUrl, {
+    username: device.username,
+    password: device.password,
+    clientId: `qris-acl-smoke-${device.username}-${Date.now()}`,
+    reconnectPeriod: 0,
+    connectTimeout: timeoutMs
+  });
+
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      client.end(true);
+      reject(new Error(`connect timeout for ${device.username}`));
+    }, timeoutMs);
+    client.once("connect", () => {
+      clearTimeout(timer);
+      resolve(client);
+    });
+    client.once("error", (error) => {
+      clearTimeout(timer);
+      client.end(true);
+      reject(error);
+    });
+  });
+}
+
+function subscribe(client, topic) {
+  return new Promise((resolve, reject) => {
+    client.subscribe(topic, { qos: 1 }, (error, grants) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(grants || []);
+    });
+  });
+}
+
+function publish(client, topic, payload) {
+  return new Promise((resolve, reject) => {
+    client.publish(topic, payload, { qos: 1 }, (error) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
+function granted(grants) {
+  return grants.some((grant) => Number(grant.qos) !== 128);
+}
+
+const client = await connectDevice(deviceA);
+try {
+  const ownDownlink = `devices/${deviceA.username}/downlink/#`;
+  const ownUplink = `devices/${deviceA.username}/uplink/acl-smoke`;
+  const foreignDownlink = `devices/${deviceB.username}/downlink/#`;
+
+  const ownGrants = await subscribe(client, ownDownlink);
+  if (!granted(ownGrants)) {
+    throw new Error(`device ${deviceA.username} could not subscribe own downlink topic`);
+  }
+
+  await publish(
+    client,
+    ownUplink,
+    JSON.stringify({ type: "acl_smoke", device_id: deviceA.username, timestamp: new Date().toISOString() })
+  );
+
+  const foreignGrants = await subscribe(client, foreignDownlink);
+  if (granted(foreignGrants)) {
+    throw new Error(`ACL allows ${deviceA.username} to subscribe foreign downlink ${foreignDownlink}`);
+  }
+
+  console.log(
+    JSON.stringify(
+      {
+        ok: true,
+        broker_url: brokerUrl,
+        device: deviceA.username,
+        own_downlink: ownDownlink,
+        own_uplink: ownUplink,
+        foreign_downlink_denied: foreignDownlink
+      },
+      null,
+      2
+    )
+  );
+} finally {
+  client.end(true);
+}

scripts/smoke-mqtt-real.mjs

@@ -0,0 +1,424 @@
+import { spawn } from "node:child_process";
+import { createHmac } from "node:crypto";
+import { Pool } from "pg";
+import mqtt from "mqtt";
+import "dotenv/config";
+
+const PORT = process.env.PORT || "3115";
+const BASE = process.env.BASE_URL || `http://127.0.0.1:${PORT}`;
+const ADMIN_TOKEN = process.env.ADMIN_TOKEN || "admin-dev-token";
+const DEVICE_TOKEN = process.env.DEVICE_TOKEN || "device-dev-token";
+const SECRET = process.env.INTEGRATION_WEBHOOK_SECRET || "dev-callback-secret";
+const BROKER_URL = process.env.MQTT_BROKER_URL;
+const MQTT_USERNAME = process.env.MQTT_USERNAME;
+const MQTT_PASSWORD = process.env.MQTT_PASSWORD;
+const MQTT_CLIENT_ID = `${process.env.MQTT_CLIENT_ID || "qris-platform-backend"}-real-smoke-${Date.now()}`;
+
+const created = {
+  merchantIds: [],
+  dynamicPartnerReferences: [],
+  mqttCorrelationIds: []
+};
+
+function short(data) {
+  const json = typeof data === "string" ? data : JSON.stringify(data || {});
+  return json.length > 220 ? `${json.slice(0, 220)}...` : json;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function req(path, options = {}) {
+  const response = await fetch(`${BASE}${path}`, {
+    method: options.method || "GET",
+    headers: {
+      "Content-Type": "application/json",
+      ...(options.headers || {})
+    },
+    body: Object.prototype.hasOwnProperty.call(options, "body")
+      ? JSON.stringify(options.body)
+      : undefined
+  });
+
+  const text = await response.text();
+  let body = null;
+  try {
+    body = text ? JSON.parse(text) : null;
+  } catch {
+    body = text;
+  }
+
+  if (!response.ok) {
+    throw new Error(`${options._label || path} failed: ${response.status} ${short(body)}`);
+  }
+
+  console.log(`${options._label || `${options.method || "GET"} ${path}`} => ${response.status} ${short(body)}`);
+  return body?.data !== undefined ? body.data : body;
+}
+
+function reqAdmin(path, options = {}) {
+  return req(path, {
+    ...options,
+    headers: {
+      ...(options.headers || {}),
+      Authorization: `Bearer ${ADMIN_TOKEN}`
+    }
+  });
+}
+
+function reqDevice(path, options = {}) {
+  return req(path, {
+    ...options,
+    headers: {
+      ...(options.headers || {}),
+      Authorization: `Bearer ${DEVICE_TOKEN}`
+    }
+  });
+}
+
+async function waitForHealth() {
+  for (let i = 0; i < 100; i += 1) {
+    try {
+      await req("/health", { _label: "GET /health" });
+      return;
+    } catch {
+      await sleep(200);
+    }
+  }
+  throw new Error("server health timeout");
+}
+
+function waitForMqtt(client) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error("MQTT_CONNECT_TIMEOUT")), 10000);
+    client.once("connect", () => {
+      clearTimeout(timer);
+      resolve();
+    });
+    client.once("error", (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+  });
+}
+
+function subscribe(client, topic) {
+  return new Promise((resolve, reject) => {
+    client.subscribe(topic, { qos: 1 }, (error) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
+function waitForMessage(messages, predicate, label) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error(`${label} message timeout`)), 12000);
+    const interval = setInterval(() => {
+      const found = messages.find(predicate);
+      if (found) {
+        clearInterval(interval);
+        clearTimeout(timer);
+        resolve(found);
+      }
+    }, 100);
+  });
+}
+
+async function cleanup() {
+  const pool = new Pool(
+    process.env.DATABASE_URL
+      ? { connectionString: process.env.DATABASE_URL }
+      : {
+          host: process.env.PGHOST || "127.0.0.1",
+          port: Number(process.env.PGPORT || 5432),
+          user: process.env.PGUSER || "postgres",
+          password: process.env.PGPASSWORD || "postgres",
+          database: process.env.PGDATABASE || "qris_soundbox_platform"
+        }
+  );
+
+  try {
+    const dynamic = await pool.query(
+      "DELETE FROM transactions WHERE partner_reference = ANY($1::text[]) RETURNING id",
+      [created.dynamicPartnerReferences]
+    );
+    const merchants = await pool.query(
+      "DELETE FROM merchants WHERE id = ANY($1::text[]) OR legal_name LIKE 'Real MQTT Smoke Merchant %' RETURNING id",
+      [created.merchantIds]
+    );
+    const mqttMessages = await pool.query(
+      "DELETE FROM mqtt_messages WHERE correlation_id = ANY($1::text[]) OR payload_json::text LIKE '%real-mqtt-smoke-%' RETURNING id",
+      [created.mqttCorrelationIds]
+    );
+    console.log(
+      `cleanup => dynamic_transactions=${dynamic.rowCount} merchants=${merchants.rowCount} mqtt_messages=${mqttMessages.rowCount}`
+    );
+  } finally {
+    await pool.end();
+  }
+}
+
+async function createMerchantBundle(ts, suffix, terminalMode = "static", deviceMode = "mqtt", capability = {}) {
+  const merchant = await reqAdmin("/admin/merchants", {
+    method: "POST",
+    body: {
+      legal_name: `Real MQTT Smoke Merchant ${suffix} ${ts}`,
+      brand_name: `RMS-${suffix}-${ts}`,
+      settlement_account_reference: `bank:${suffix}:${ts}`,
+      settlement_account_type: "merchant_bank_account",
+      payout_mode: "merchant_direct"
+    },
+    _label: `POST /admin/merchants ${suffix}`
+  });
+  created.merchantIds.push(merchant.id);
+
+  const outlet = await reqAdmin(`/admin/merchants/${merchant.id}/outlets`, {
+    method: "POST",
+    body: { name: `Real MQTT Smoke Outlet ${suffix} ${ts}` },
+    _label: `POST /admin/merchants/:id/outlets ${suffix}`
+  });
+
+  const terminal = await reqAdmin(`/admin/outlets/${outlet.id}/terminals`, {
+    method: "POST",
+    body: { terminal_code: `RMS-${suffix}-TERM-${ts}`, qr_mode: terminalMode },
+    _label: `POST /admin/outlets/:id/terminals ${suffix}`
+  });
+
+  const device = await reqAdmin("/admin/devices", {
+    method: "POST",
+    body: {
+      device_code: `RMS-${suffix}-DEV-${ts}`,
+      vendor: "smoke",
+      model: "mqtt-real",
+      communication_mode: deviceMode,
+      status: "active",
+      capability_profile_json: capability
+    },
+    _label: `POST /admin/devices ${suffix}`
+  });
+
+  await reqAdmin(`/admin/devices/${device.id}/bind`, {
+    method: "POST",
+    body: {
+      merchant_id: merchant.id,
+      outlet_id: outlet.id,
+      terminal_id: terminal.id
+    },
+    _label: `POST /admin/devices/:id/bind ${suffix}`
+  });
+
+  return { merchant, outlet, terminal, device };
+}
+
+async function main() {
+  if (!BROKER_URL || !MQTT_USERNAME || !MQTT_PASSWORD) {
+    throw new Error("MQTT_BROKER_URL, MQTT_USERNAME, and MQTT_PASSWORD are required");
+  }
+
+  const server = spawn("npm", ["start"], {
+    cwd: process.cwd(),
+    env: {
+      ...process.env,
+      PORT,
+      MQTT_PUBLISH_MODE: "broker"
+    },
+    stdio: ["ignore", "pipe", "pipe"]
+  });
+
+  const serverLog = [];
+  server.stdout.on("data", (chunk) => serverLog.push(chunk.toString()));
+  server.stderr.on("data", (chunk) => serverLog.push(chunk.toString()));
+
+  const client = mqtt.connect(BROKER_URL, {
+    clientId: MQTT_CLIENT_ID,
+    username: MQTT_USERNAME,
+    password: MQTT_PASSWORD,
+    connectTimeout: 10000,
+    reconnectPeriod: 0,
+    clean: true
+  });
+
+  const messages = [];
+  client.on("message", (topic, payload) => {
+    let parsed;
+    try {
+      parsed = JSON.parse(payload.toString());
+    } catch {
+      parsed = payload.toString();
+    }
+    messages.push({ topic, payload: parsed });
+  });
+
+  try {
+    await Promise.all([waitForHealth(), waitForMqtt(client)]);
+    await subscribe(client, "devices/+/downlink/#");
+    console.log(`MQTT subscribe => devices/+/downlink/#`);
+
+    const ts = Date.now();
+
+    const staticBundle = await createMerchantBundle(ts, "PAY", "static", "mqtt", {
+      flows: ["static_payment_notification"]
+    });
+    await reqDevice("/device/heartbeat", {
+      method: "POST",
+      body: {
+        device_id: staticBundle.device.id,
+        timestamp: new Date().toISOString(),
+        network_strength: 88,
+        battery_level: 77,
+        state: "idle"
+      },
+      _label: "POST /device/heartbeat payment"
+    });
+
+    const partnerReference = `REAL-MQTT-SMOKE-PR-${ts}`;
+    const paymentTx = await reqAdmin("/admin/transactions", {
+      method: "POST",
+      body: {
+        partner_reference: partnerReference,
+        merchant_id: staticBundle.merchant.id,
+        outlet_id: staticBundle.outlet.id,
+        terminal_id: staticBundle.terminal.id,
+        device_id: staticBundle.device.id,
+        amount: 32100,
+        currency: "IDR",
+        qr_mode: "static",
+        initiation_mode: "static",
+        status: "initiated"
+      },
+      _label: "POST /admin/transactions payment"
+    });
+
+    const callback = {
+      partner_reference: partnerReference,
+      partner_txn_id: `REAL-MQTT-SMOKE-PTX-${ts}`,
+      amount: 32100,
+      currency: "IDR",
+      payment_status: "paid",
+      status: "paid",
+      paid_at: new Date().toISOString()
+    };
+    const signature = createHmac("sha256", SECRET).update(JSON.stringify(callback)).digest("hex");
+    await req("/integrations/qris/callback", {
+      method: "POST",
+      headers: { "X-Partner-Signature": signature },
+      body: { ...callback, signature },
+      _label: "POST /integrations/qris/callback payment"
+    });
+
+    const paymentMessage = await waitForMessage(
+      messages,
+      (item) =>
+        item.topic === `devices/${staticBundle.device.id}/downlink/payment/success` &&
+        item.payload?.message_type === "payment_success" &&
+        item.payload?.transaction_id === paymentTx.id,
+      "payment success"
+    );
+
+    const apiBundle = await createMerchantBundle(ts, "CFG", "dynamic_api", "api", {
+      features: { dynamic_qr: { api_direct: true } },
+      flows: ["dynamic_qr:api_direct"]
+    });
+    const configVersion = Math.floor(Date.now() / 1000);
+    await reqAdmin(`/admin/devices/${apiBundle.device.id}/config`, {
+      method: "PATCH",
+      body: {
+        config_version: configVersion,
+        settings: {
+          volume: 62,
+          language: "id-ID",
+          heartbeat_interval_seconds: 45,
+          test_marker: `real-mqtt-smoke-${configVersion}`
+        }
+      },
+      _label: "PATCH /admin/devices/:id/config"
+    });
+
+    const configMessage = await waitForMessage(
+      messages,
+      (item) =>
+        item.topic === `devices/${apiBundle.device.id}/downlink/config/push` &&
+        item.payload?.message_type === "config_push" &&
+        item.payload?.config_version === configVersion,
+      "config push"
+    );
+
+    const mqttBundle = await createMerchantBundle(ts, "DYN", "dynamic_mqtt", "mqtt", {
+      features: { dynamic_qr: { mqtt: true } },
+      flows: ["dynamic_qr:mqtt"]
+    });
+    const requestId = `REAL-MQTT-SMOKE-DYN-${ts}`;
+    created.mqttCorrelationIds.push(requestId);
+    await reqDevice("/device/mqtt/uplink/dynamic-qr/request", {
+      method: "POST",
+      body: {
+        message_type: "dynamic_qr_request",
+        device_id: mqttBundle.device.id,
+        terminal_id: mqttBundle.terminal.id,
+        amount: 12345,
+        currency: "IDR",
+        request_id: requestId
+      },
+      _label: "POST /device/mqtt/uplink/dynamic-qr/request"
+    });
+    created.dynamicPartnerReferences.push(`DYN-${requestId}`);
+
+    const dynamicMessage = await waitForMessage(
+      messages,
+      (item) =>
+        item.topic === `devices/${mqttBundle.device.id}/downlink/dynamic-qr/response` &&
+        item.payload?.message_type === "dynamic_qr_response" &&
+        item.payload?.correlation_id === requestId,
+      "dynamic QR response"
+    );
+
+    console.log(
+      JSON.stringify(
+        {
+          broker_connect: "ok",
+          subscribed_topic: "devices/+/downlink/#",
+          payment_success: {
+            topic: paymentMessage.topic,
+            transaction_id: paymentMessage.payload.transaction_id,
+            amount: paymentMessage.payload.amount
+          },
+          config_push: {
+            topic: configMessage.topic,
+            config_version: configMessage.payload.config_version
+          },
+          dynamic_qr_response: {
+            topic: dynamicMessage.topic,
+            transaction_id: dynamicMessage.payload.transaction_id,
+            correlation_id: dynamicMessage.payload.correlation_id
+          },
+          received_messages_count: messages.length
+        },
+        null,
+        2
+      )
+    );
+  } finally {
+    client.end(true);
+    server.kill("SIGTERM");
+    await sleep(500);
+    await cleanup();
+    if (server.exitCode === null) {
+      server.kill("SIGKILL");
+    }
+    if (serverLog.length) {
+      process.env.SMOKE_MQTT_REAL_DEBUG === "true" && console.log(serverLog.join(""));
+    }
+  }
+  process.exit(0);
+}
+
+main().catch(async (error) => {
+  console.error(error instanceof Error ? error.message : error);
+  await cleanup().catch(() => undefined);
+  process.exit(1);
+});

scripts/smoke.mjs

@@ -3,6 +3,8 @@ import { createHmac } from "node:crypto";
 const PORT = process.env.PORT || "3100";
 const BASE = process.env.BASE_URL || `http://127.0.0.1:${PORT}`;
 const ADMIN_TOKEN = process.env.ADMIN_TOKEN || "admin-dev-token";
+const MERCHANT_TOKEN = process.env.MERCHANT_TOKEN || "merchant-dev-token";
+const MERCHANT_PORTAL_PASSWORD = process.env.MERCHANT_PORTAL_PASSWORD || "merchant";
 const DEVICE_TOKEN = process.env.DEVICE_TOKEN || "device-dev-token";
 const SECRET = process.env.INTEGRATION_WEBHOOK_SECRET || "dev-callback-secret";
 
@@ -67,10 +69,32 @@ async function reqAdmin(path, opts = {}) {
   return req(path, { ...opts, headers: { ...(opts.headers || {}), Authorization: `Bearer ${ADMIN_TOKEN}` } });
 }
 
+async function reqMerchant(path, merchantId, opts = {}) {
+  return req(path, {
+    ...opts,
+    headers: {
+      ...(opts.headers || {}),
+      Authorization: `Bearer ${MERCHANT_TOKEN}`,
+      'X-Merchant-Id': merchantId
+    }
+  });
+}
+
 async function reqDevice(path, opts = {}) {
   return req(path, { ...opts, headers: { ...(opts.headers || {}), Authorization: `Bearer ${DEVICE_TOKEN}` } });
 }
 
+async function reqDeviceCredential(path, deviceId, secret, opts = {}) {
+  return req(path, {
+    ...opts,
+    headers: {
+      ...(opts.headers || {}),
+      'X-Device-Id': deviceId,
+      'X-Device-Secret': secret
+    }
+  });
+}
+
 (async () => {
   await req('/health', { _label: 'GET /health' });
   await req('/admin/login', { method: 'POST', body: { username: 'admin', password: 'admin' }, _label: 'POST /admin/login' });
@@ -141,6 +165,28 @@ async function reqDevice(path, opts = {}) {
     _label: 'POST /device/heartbeat'
   });
 
+  const credential = await reqAdmin(`/admin/devices/${deviceId}/credentials/rotate`, {
+    method: 'POST',
+    body: {},
+    _label: 'POST /admin/devices/:id/credentials/rotate'
+  });
+  const deviceSecret = credential?.data?.credential?.mqtt_password;
+  if (!deviceSecret) {
+    throw new Error('device credential rotate did not return one-time password');
+  }
+  await reqDeviceCredential('/device/heartbeat', deviceId, deviceSecret, {
+    method: 'POST',
+    body: {
+      device_id: deviceId,
+      timestamp: new Date().toISOString(),
+      firmware_version: '1.2.4',
+      network_strength: 89,
+      battery_level: 76,
+      state: 'credential-auth'
+    },
+    _label: 'POST /device/heartbeat credential auth'
+  });
+
   const tx = await reqAdmin('/admin/transactions', {
     method: 'POST',
     body: {
@@ -194,6 +240,163 @@ async function reqDevice(path, opts = {}) {
   await reqAdmin(`/admin/transactions/${txId}`, { _label: 'GET /admin/transactions/:id' });
   await reqAdmin(`/admin/transactions/${txId}/events`, { _label: 'GET /admin/transactions/:id/events' });
   await reqAdmin(`/admin/ledger-entries?transaction_id=${txId}`, { _label: 'GET /admin/ledger-entries' });
+  const settlementBatchCreate = await reqAdmin('/admin/settlement-batches', {
+    method: 'POST',
+    body: {
+      merchant_id: merchantId,
+      cutoff_at: new Date().toISOString()
+    },
+    _label: 'POST /admin/settlement-batches'
+  });
+  const settlementBatch = settlementBatchCreate?.data?.batches?.[0];
+  if (!settlementBatch || settlementBatch.entry_count < 1 || Number(settlementBatch.net_payable_amount) <= 0) {
+    throw new Error('settlement batch did not include merchant payable entries');
+  }
+  await reqAdmin('/admin/settlement-batches?status=created', { _label: 'GET /admin/settlement-batches' });
+  await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}`, { _label: 'GET /admin/settlement-batches/:id' });
+  const settlementCsv = await req(`/admin/settlement-batches/${settlementBatch.id}/export.csv`, {
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'GET /admin/settlement-batches/:id/export.csv'
+  });
+  if (!String(settlementCsv).includes('batch_code,batch_status,merchant_id')) {
+    throw new Error('settlement CSV export missing expected header');
+  }
+  const settlementBankCsv = await req(`/admin/settlement-batches/${settlementBatch.id}/export.csv?format=bank_generic`, {
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'GET /admin/settlement-batches/:id/export.csv bank_generic'
+  });
+  if (!String(settlementBankCsv).includes('transfer_type,beneficiary_account_reference,beneficiary_account_type')) {
+    throw new Error('settlement bank generic CSV export missing expected header');
+  }
+  await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}/mark-paid`, {
+    method: 'POST',
+    body: {
+      paid_at: new Date().toISOString(),
+      paid_reference: `SMOKE-PAYOUT-${ts}`,
+      paid_note: 'Smoke payout reconciliation note'
+    },
+    _label: 'POST /admin/settlement-batches/:id/mark-paid'
+  });
+  const settlementDetailWithEvents = await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id events'
+  });
+  const settlementEvents = settlementDetailWithEvents?.data?.events || [];
+  if (!settlementEvents.some((event) => event.event_type === 'created') || !settlementEvents.some((event) => event.event_type === 'marked_paid')) {
+    throw new Error('settlement payout event history missing created or marked_paid event');
+  }
+  await reqExpect(`/admin/settlement-batches/${settlementBatch.id}/mark-paid`, 409, {
+    method: 'POST',
+    body: {},
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'POST /admin/settlement-batches/:id/mark-paid duplicate'
+  });
+  await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}/reference`, {
+    method: 'PATCH',
+    body: {
+      paid_reference: `SMOKE-PAYOUT-UPDATED-${ts}`,
+      paid_note: 'Smoke payout reference correction'
+    },
+    _label: 'PATCH /admin/settlement-batches/:id/reference'
+  });
+  const settlementDetailAfterReference = await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id reference-updated'
+  });
+  if (
+    settlementDetailAfterReference?.data?.batch?.metadata_json?.paid_reference !== `SMOKE-PAYOUT-UPDATED-${ts}` ||
+    !settlementDetailAfterReference?.data?.events?.some((event) => event.event_type === 'reference_updated')
+  ) {
+    throw new Error('settlement reference update did not persist metadata or event history');
+  }
+  await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}/adjustments`, {
+    method: 'POST',
+    body: {
+      adjustment_type: 'debit',
+      amount: 100,
+      reason: 'Smoke reconciliation fee correction',
+      note: 'Smoke adjustment event'
+    },
+    _label: 'POST /admin/settlement-batches/:id/adjustments'
+  });
+  const settlementDetailAfterAdjustment = await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id adjustment-recorded'
+  });
+  if (
+    Number(settlementDetailAfterAdjustment?.data?.batch?.metadata_json?.total_adjustment_amount || 0) !== -100 ||
+    !Array.isArray(settlementDetailAfterAdjustment?.data?.adjustments) ||
+    settlementDetailAfterAdjustment.data.adjustments.length < 1 ||
+    Number(settlementDetailAfterAdjustment.data.adjustments[0]?.signed_amount || 0) !== -100 ||
+    !settlementDetailAfterAdjustment?.data?.events?.some((event) => event.event_type === 'adjustment_recorded')
+  ) {
+    throw new Error('settlement adjustment did not persist metadata or event history');
+  }
+  const settlementAdjustmentReport = await reqAdmin(`/admin/settlement-adjustments?merchant_id=${merchantId}&limit=20`, {
+    _label: 'GET /admin/settlement-adjustments'
+  });
+  if (
+    !Array.isArray(settlementAdjustmentReport?.data?.rows) ||
+    settlementAdjustmentReport.data.rows.length < 1 ||
+    Number(settlementAdjustmentReport.data.signed_amount || 0) !== -100 ||
+    !settlementAdjustmentReport.data.rows.some((row) => row.batch_id === settlementBatch.id && Number(row.signed_amount || 0) === -100)
+  ) {
+    throw new Error('settlement adjustment report missing expected row or totals');
+  }
+  const settlementAdjustmentCsv = await req(`/admin/settlement-adjustments/export.csv?merchant_id=${merchantId}&limit=20`, {
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'GET /admin/settlement-adjustments/export.csv'
+  });
+  if (
+    !String(settlementAdjustmentCsv).includes('adjustment_id,batch_id,batch_code') ||
+    !String(settlementAdjustmentCsv).includes('Smoke reconciliation fee correction')
+  ) {
+    throw new Error('settlement adjustment CSV export missing expected header or row');
+  }
+  const merchantLogin = await req('/merchant/login', {
+    method: 'POST',
+    body: { username: merchantId, password: MERCHANT_PORTAL_PASSWORD },
+    _label: 'POST /merchant/login'
+  });
+  if (merchantLogin?.data?.merchant?.id !== merchantId || !merchantLogin?.data?.token) {
+    throw new Error('merchant login did not return expected merchant session');
+  }
+  const merchantSummary = await reqMerchant('/merchant/settlement-summary', merchantId, {
+    _label: 'GET /merchant/settlement-summary'
+  });
+  const merchantPaidAmount = Number(merchantSummary?.data?.paid_amount || 0);
+  const merchantAdjustmentAmount = Number(merchantSummary?.data?.adjustment_amount || 0);
+  const merchantAdjustedPaidAmount = Number(merchantSummary?.data?.adjusted_paid_amount || 0);
+  if (
+    merchantPaidAmount < Number(settlementBatch.net_payable_amount || 0) ||
+    merchantAdjustmentAmount !== -100 ||
+    Math.abs(merchantAdjustedPaidAmount - (merchantPaidAmount - 100)) > 0.01
+  ) {
+    throw new Error('merchant settlement summary missing paid amount');
+  }
+  await reqMerchant('/merchant/settlement-batches', merchantId, {
+    _label: 'GET /merchant/settlement-batches'
+  });
+  const merchantBatchDetail = await reqMerchant(`/merchant/settlement-batches/${settlementBatch.id}`, merchantId, {
+    _label: 'GET /merchant/settlement-batches/:id'
+  });
+  if (!merchantBatchDetail?.data?.events?.some((event) => event.event_type === 'marked_paid')) {
+    throw new Error('merchant settlement batch detail missing payout event history');
+  }
+  if (!Array.isArray(merchantBatchDetail?.data?.adjustments) || merchantBatchDetail.data.adjustments.length < 1) {
+    throw new Error('merchant settlement batch detail missing formal adjustment rows');
+  }
+  const merchantSettlementCsv = await req(`/merchant/settlement-batches/${settlementBatch.id}/export.csv`, {
+    headers: { Authorization: `Bearer ${MERCHANT_TOKEN}`, 'X-Merchant-Id': merchantId },
+    _label: 'GET /merchant/settlement-batches/:id/export.csv'
+  });
+  if (!String(merchantSettlementCsv).includes('batch_code,batch_status,merchant_id')) {
+    throw new Error('merchant settlement CSV export missing expected header');
+  }
+  const merchantSettlementBankCsv = await req(`/merchant/settlement-batches/${settlementBatch.id}/export.csv?format=bank_generic`, {
+    headers: { Authorization: `Bearer ${MERCHANT_TOKEN}`, 'X-Merchant-Id': merchantId },
+    _label: 'GET /merchant/settlement-batches/:id/export.csv bank_generic'
+  });
+  if (!String(merchantSettlementBankCsv).includes('transfer_type,beneficiary_account_reference,beneficiary_account_type')) {
+    throw new Error('merchant settlement bank generic CSV export missing expected header');
+  }
   await reqAdmin(`/admin/audit-logs?entity_id=${txId}`, { _label: 'GET /admin/audit-logs' });
   await reqAdmin(`/admin/transactions/${txId}/heartbeats`, { _label: 'GET /admin/transactions/:id/heartbeats' });
   await reqAdmin(`/admin/devices/${deviceId}/heartbeats`, { _label: 'GET /admin/devices/:id/heartbeats' });
@@ -204,7 +407,20 @@ async function reqDevice(path, opts = {}) {
     body: {},
     _label: 'POST /admin/transactions/:id/retry-notification'
   });
-  await reqAdmin('/admin/dashboard/summary', { _label: 'GET /admin/dashboard/summary' });
+  const dashboardSummary = await reqAdmin('/admin/dashboard/summary', { _label: 'GET /admin/dashboard/summary' });
+  const dashboardData = dashboardSummary?.data || {};
+  const dashboardPaidAmount = Number(dashboardData.settlement_paid_amount || 0);
+  const dashboardAdjustmentAmount = Number(dashboardData.settlement_adjustment_amount || 0);
+  const dashboardAdjustedPaidAmount = Number(dashboardData.settlement_adjusted_paid_amount || 0);
+  if (
+    dashboardPaidAmount < Number(settlementBatch.net_payable_amount || 0) ||
+    dashboardAdjustmentAmount !== -100 ||
+    Math.abs(dashboardAdjustedPaidAmount - (dashboardPaidAmount - 100)) > 0.01 ||
+    Number(dashboardData.settlement_paid_batches || 0) < 1 ||
+    Number(dashboardData.settlement_total_batches || 0) < 1
+  ) {
+    throw new Error('dashboard settlement finance summary missing paid settlement aggregate');
+  }
 
   const noBindingOutlet = await reqAdmin(`/admin/merchants/${merchantId}/outlets`, {
     method: 'POST',
@@ -253,6 +469,129 @@ async function reqDevice(path, opts = {}) {
     _label: 'GET /admin/ledger-entries no-binding'
   });
   await reqAdmin('/admin/notifications/failed', { _label: 'GET /admin/notifications/failed no-binding' });
+  const failedBatchCreate = await reqAdmin('/admin/settlement-batches', {
+    method: 'POST',
+    body: {
+      merchant_id: merchantId,
+      cutoff_at: new Date().toISOString()
+    },
+    _label: 'POST /admin/settlement-batches failed-case'
+  });
+  const failedBatch = failedBatchCreate?.data?.batches?.[0];
+  if (!failedBatch) {
+    throw new Error('failed settlement case did not create a batch');
+  }
+  await reqAdmin(`/admin/settlement-batches/${failedBatch.id}/mark-failed`, {
+    method: 'POST',
+    body: {
+      reason: 'Smoke payout rail rejected transfer',
+      note: 'Smoke failed settlement lifecycle'
+    },
+    _label: 'POST /admin/settlement-batches/:id/mark-failed'
+  });
+  const failedBatchDetail = await reqAdmin(`/admin/settlement-batches/${failedBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id failed'
+  });
+  if (
+    failedBatchDetail?.data?.batch?.status !== 'failed' ||
+    !failedBatchDetail?.data?.events?.some((event) => event.event_type === 'failed')
+  ) {
+    throw new Error('settlement failed lifecycle did not persist failed status/event');
+  }
+  const reprocessedFailed = await reqAdmin(`/admin/settlement-batches/${failedBatch.id}/reprocess`, {
+    method: 'POST',
+    body: {},
+    _label: 'POST /admin/settlement-batches/:id/reprocess failed'
+  });
+  const reprocessedFailedBatch = reprocessedFailed?.data?.new_batch;
+  if (!reprocessedFailedBatch || reprocessedFailedBatch.status !== 'created') {
+    throw new Error('failed settlement reprocess did not create a new created batch');
+  }
+  const reprocessedFailedSource = await reqAdmin(`/admin/settlement-batches/${failedBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id reprocessed-source'
+  });
+  if (!reprocessedFailedSource?.data?.events?.some((event) => event.event_type === 'reprocessed')) {
+    throw new Error('failed settlement reprocess did not create source reprocessed event');
+  }
+  await reqExpect(`/admin/settlement-batches/${failedBatch.id}/reprocess`, 409, {
+    method: 'POST',
+    body: {},
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'POST /admin/settlement-batches/:id/reprocess duplicate'
+  });
+
+  const cancelTx = await reqAdmin('/admin/transactions', {
+    method: 'POST',
+    body: {
+      partner_reference: `PR-CANCEL-${ts}`,
+      merchant_id: merchantId,
+      outlet_id: outletId,
+      terminal_id: terminalId,
+      amount: 7700,
+      currency: 'IDR',
+      qr_mode: 'static',
+      initiation_mode: 'static',
+      status: 'initiated'
+    },
+    _label: 'POST /admin/transactions cancel-settlement'
+  });
+  const cancelCallback = {
+    partner_reference: `PR-CANCEL-${ts}`,
+    partner_txn_id: `PTX-CANCEL-${ts}`,
+    amount: 7700,
+    currency: 'IDR',
+    payment_status: 'paid',
+    status: 'paid',
+    paid_at: new Date().toISOString()
+  };
+  const cancelSignature = createHmac('sha256', SECRET).update(JSON.stringify(cancelCallback)).digest('hex');
+  await req('/integrations/qris/callback', {
+    method: 'POST',
+    headers: { 'X-Partner-Signature': cancelSignature },
+    body: { ...cancelCallback, signature: cancelSignature },
+    _label: 'POST /integrations/qris/callback cancel-settlement'
+  });
+  const cancelBatchCreate = await reqAdmin('/admin/settlement-batches', {
+    method: 'POST',
+    body: {
+      merchant_id: merchantId,
+      cutoff_at: new Date().toISOString()
+    },
+    _label: 'POST /admin/settlement-batches cancel-case'
+  });
+  const cancelBatch = cancelBatchCreate?.data?.batches?.[0];
+  if (!cancelBatch) {
+    throw new Error('cancel settlement case did not create a batch');
+  }
+  await reqAdmin(`/admin/settlement-batches/${cancelBatch.id}/cancel`, {
+    method: 'POST',
+    body: {
+      reason: 'Smoke duplicate manual payout batch',
+      note: `cancel tx ${cancelTx?.data?.id}`
+    },
+    _label: 'POST /admin/settlement-batches/:id/cancel'
+  });
+  const cancelBatchDetail = await reqAdmin(`/admin/settlement-batches/${cancelBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id cancelled'
+  });
+  if (
+    cancelBatchDetail?.data?.batch?.status !== 'cancelled' ||
+    !cancelBatchDetail?.data?.events?.some((event) => event.event_type === 'cancelled')
+  ) {
+    throw new Error('settlement cancel lifecycle did not persist cancelled status/event');
+  }
+  const reconciliationReport = await reqAdmin('/admin/reconciliation/settlement-batches?limit=50', {
+    _label: 'GET /admin/reconciliation/settlement-batches'
+  });
+  const reconciliationData = reconciliationReport?.data || {};
+  if (
+    !Array.isArray(reconciliationData.rows) ||
+    !Number.isFinite(Number(reconciliationData.total_batches)) ||
+    !Number.isFinite(Number(reconciliationData.mismatch_batches)) ||
+    Number(reconciliationData.total_batches) < 1
+  ) {
+    throw new Error('settlement reconciliation report missing expected aggregate rows');
+  }
 
   const dynamicOutlet = await reqAdmin(`/admin/merchants/${merchantId}/outlets`, {
     method: 'POST',
@@ -303,6 +642,21 @@ async function reqDevice(path, opts = {}) {
     },
     _label: 'POST /device/transactions/dynamic-qr unsupported device'
   });
+  await reqExpect('/device/heartbeat', 403, {
+    method: 'POST',
+    headers: {
+      'X-Device-Id': deviceId,
+      'X-Device-Secret': deviceSecret
+    },
+    body: {
+      device_id: dynamicDeviceId,
+      timestamp: new Date().toISOString(),
+      network_strength: 80,
+      battery_level: 70,
+      state: 'wrong-device'
+    },
+    _label: 'POST /device/heartbeat credential wrong device'
+  });
   const dynamicRequestId = `DYN-REQ-${ts}`;
   const dynamicQr = await reqDevice('/device/transactions/dynamic-qr', {
     method: 'POST',

scripts/ui-qa-check.mjs

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+
+const pages = [
+  "ui/admin-reconciliation-management/index.html",
+  "ui/admin-system-audit-logs/index.html",
+  "ui/settlement-batch-management/index.html",
+  "ui/merchant-settlement-history/index.html",
+  "ui/device-technical-detail/index.html"
+];
+
+const checks = [];
+const warnings = [];
+const errors = [];
+const scriptRe = new RegExp("<script(?:\\s[^>]*)?>([\\s\\S]*?)</script>", "gi");
+
+function check(condition, message) {
+  checks.push({ ok: Boolean(condition), message });
+  if (!condition) {
+    errors.push(message);
+  }
+}
+
+function warn(condition, message) {
+  if (!condition) {
+    warnings.push(message);
+  }
+}
+
+for (const page of pages) {
+  const filePath = path.resolve(process.cwd(), page);
+  const html = fs.readFileSync(filePath, "utf8");
+  const scripts = [...html.matchAll(scriptRe)].map((match) => match[1].trim()).filter(Boolean);
+  for (const [index, script] of scripts.entries()) {
+    try {
+      new Function(script);
+      check(true, `${page} script ${index + 1} parses`);
+    } catch (error) {
+      check(false, `${page} script ${index + 1} parse failed: ${error.message}`);
+    }
+  }
+  warn(!/href="#"/.test(html), `${page} still has placeholder # links for manual navigation QA`);
+}
+
+const reconciliation = fs.readFileSync("ui/admin-reconciliation-management/index.html", "utf8");
+check(reconciliation.includes("adjustment-export-history"), "reconciliation export history is present");
+check(!reconciliation.includes("/admin/settlement-adjustments/export.csv?"), "reconciliation uses async export instead of sync CSV URL");
+
+const settlement = fs.readFileSync("ui/settlement-batch-management/index.html", "utf8");
+check(settlement.includes('data-admin-permission="settlement:export"'), "settlement export button is permission-aware");
+check(settlement.includes('data-admin-permission="settlement:pay"'), "settlement pay action is permission-aware");
+
+const auditLogs = fs.readFileSync("ui/admin-system-audit-logs/index.html", "utf8");
+check(auditLogs.includes("audit-action-filter"), "audit logs page has action filter");
+check(auditLogs.includes("admin.login.failed"), "audit logs page supports admin login failed filter");
+check(auditLogs.includes("merchant.login.failed"), "audit logs page supports merchant login failed filter");
+
+console.log(JSON.stringify({ ok: errors.length === 0, checks, warnings, errors }, null, 2));
+process.exit(errors.length ? 1 : 0);