Production readiness hardening and ops tooling

scripts/smoke.mjs

@@ -3,6 +3,8 @@ import { createHmac } from "node:crypto";
 const PORT = process.env.PORT || "3100";
 const BASE = process.env.BASE_URL || `http://127.0.0.1:${PORT}`;
 const ADMIN_TOKEN = process.env.ADMIN_TOKEN || "admin-dev-token";
+const MERCHANT_TOKEN = process.env.MERCHANT_TOKEN || "merchant-dev-token";
+const MERCHANT_PORTAL_PASSWORD = process.env.MERCHANT_PORTAL_PASSWORD || "merchant";
 const DEVICE_TOKEN = process.env.DEVICE_TOKEN || "device-dev-token";
 const SECRET = process.env.INTEGRATION_WEBHOOK_SECRET || "dev-callback-secret";
 
@@ -67,10 +69,32 @@ async function reqAdmin(path, opts = {}) {
   return req(path, { ...opts, headers: { ...(opts.headers || {}), Authorization: `Bearer ${ADMIN_TOKEN}` } });
 }
 
+async function reqMerchant(path, merchantId, opts = {}) {
+  return req(path, {
+    ...opts,
+    headers: {
+      ...(opts.headers || {}),
+      Authorization: `Bearer ${MERCHANT_TOKEN}`,
+      'X-Merchant-Id': merchantId
+    }
+  });
+}
+
 async function reqDevice(path, opts = {}) {
   return req(path, { ...opts, headers: { ...(opts.headers || {}), Authorization: `Bearer ${DEVICE_TOKEN}` } });
 }
 
+async function reqDeviceCredential(path, deviceId, secret, opts = {}) {
+  return req(path, {
+    ...opts,
+    headers: {
+      ...(opts.headers || {}),
+      'X-Device-Id': deviceId,
+      'X-Device-Secret': secret
+    }
+  });
+}
+
 (async () => {
   await req('/health', { _label: 'GET /health' });
   await req('/admin/login', { method: 'POST', body: { username: 'admin', password: 'admin' }, _label: 'POST /admin/login' });
@@ -141,6 +165,28 @@ async function reqDevice(path, opts = {}) {
     _label: 'POST /device/heartbeat'
   });
 
+  const credential = await reqAdmin(`/admin/devices/${deviceId}/credentials/rotate`, {
+    method: 'POST',
+    body: {},
+    _label: 'POST /admin/devices/:id/credentials/rotate'
+  });
+  const deviceSecret = credential?.data?.credential?.mqtt_password;
+  if (!deviceSecret) {
+    throw new Error('device credential rotate did not return one-time password');
+  }
+  await reqDeviceCredential('/device/heartbeat', deviceId, deviceSecret, {
+    method: 'POST',
+    body: {
+      device_id: deviceId,
+      timestamp: new Date().toISOString(),
+      firmware_version: '1.2.4',
+      network_strength: 89,
+      battery_level: 76,
+      state: 'credential-auth'
+    },
+    _label: 'POST /device/heartbeat credential auth'
+  });
+
   const tx = await reqAdmin('/admin/transactions', {
     method: 'POST',
     body: {
@@ -194,6 +240,163 @@ async function reqDevice(path, opts = {}) {
   await reqAdmin(`/admin/transactions/${txId}`, { _label: 'GET /admin/transactions/:id' });
   await reqAdmin(`/admin/transactions/${txId}/events`, { _label: 'GET /admin/transactions/:id/events' });
   await reqAdmin(`/admin/ledger-entries?transaction_id=${txId}`, { _label: 'GET /admin/ledger-entries' });
+  const settlementBatchCreate = await reqAdmin('/admin/settlement-batches', {
+    method: 'POST',
+    body: {
+      merchant_id: merchantId,
+      cutoff_at: new Date().toISOString()
+    },
+    _label: 'POST /admin/settlement-batches'
+  });
+  const settlementBatch = settlementBatchCreate?.data?.batches?.[0];
+  if (!settlementBatch || settlementBatch.entry_count < 1 || Number(settlementBatch.net_payable_amount) <= 0) {
+    throw new Error('settlement batch did not include merchant payable entries');
+  }
+  await reqAdmin('/admin/settlement-batches?status=created', { _label: 'GET /admin/settlement-batches' });
+  await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}`, { _label: 'GET /admin/settlement-batches/:id' });
+  const settlementCsv = await req(`/admin/settlement-batches/${settlementBatch.id}/export.csv`, {
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'GET /admin/settlement-batches/:id/export.csv'
+  });
+  if (!String(settlementCsv).includes('batch_code,batch_status,merchant_id')) {
+    throw new Error('settlement CSV export missing expected header');
+  }
+  const settlementBankCsv = await req(`/admin/settlement-batches/${settlementBatch.id}/export.csv?format=bank_generic`, {
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'GET /admin/settlement-batches/:id/export.csv bank_generic'
+  });
+  if (!String(settlementBankCsv).includes('transfer_type,beneficiary_account_reference,beneficiary_account_type')) {
+    throw new Error('settlement bank generic CSV export missing expected header');
+  }
+  await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}/mark-paid`, {
+    method: 'POST',
+    body: {
+      paid_at: new Date().toISOString(),
+      paid_reference: `SMOKE-PAYOUT-${ts}`,
+      paid_note: 'Smoke payout reconciliation note'
+    },
+    _label: 'POST /admin/settlement-batches/:id/mark-paid'
+  });
+  const settlementDetailWithEvents = await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id events'
+  });
+  const settlementEvents = settlementDetailWithEvents?.data?.events || [];
+  if (!settlementEvents.some((event) => event.event_type === 'created') || !settlementEvents.some((event) => event.event_type === 'marked_paid')) {
+    throw new Error('settlement payout event history missing created or marked_paid event');
+  }
+  await reqExpect(`/admin/settlement-batches/${settlementBatch.id}/mark-paid`, 409, {
+    method: 'POST',
+    body: {},
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'POST /admin/settlement-batches/:id/mark-paid duplicate'
+  });
+  await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}/reference`, {
+    method: 'PATCH',
+    body: {
+      paid_reference: `SMOKE-PAYOUT-UPDATED-${ts}`,
+      paid_note: 'Smoke payout reference correction'
+    },
+    _label: 'PATCH /admin/settlement-batches/:id/reference'
+  });
+  const settlementDetailAfterReference = await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id reference-updated'
+  });
+  if (
+    settlementDetailAfterReference?.data?.batch?.metadata_json?.paid_reference !== `SMOKE-PAYOUT-UPDATED-${ts}` ||
+    !settlementDetailAfterReference?.data?.events?.some((event) => event.event_type === 'reference_updated')
+  ) {
+    throw new Error('settlement reference update did not persist metadata or event history');
+  }
+  await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}/adjustments`, {
+    method: 'POST',
+    body: {
+      adjustment_type: 'debit',
+      amount: 100,
+      reason: 'Smoke reconciliation fee correction',
+      note: 'Smoke adjustment event'
+    },
+    _label: 'POST /admin/settlement-batches/:id/adjustments'
+  });
+  const settlementDetailAfterAdjustment = await reqAdmin(`/admin/settlement-batches/${settlementBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id adjustment-recorded'
+  });
+  if (
+    Number(settlementDetailAfterAdjustment?.data?.batch?.metadata_json?.total_adjustment_amount || 0) !== -100 ||
+    !Array.isArray(settlementDetailAfterAdjustment?.data?.adjustments) ||
+    settlementDetailAfterAdjustment.data.adjustments.length < 1 ||
+    Number(settlementDetailAfterAdjustment.data.adjustments[0]?.signed_amount || 0) !== -100 ||
+    !settlementDetailAfterAdjustment?.data?.events?.some((event) => event.event_type === 'adjustment_recorded')
+  ) {
+    throw new Error('settlement adjustment did not persist metadata or event history');
+  }
+  const settlementAdjustmentReport = await reqAdmin(`/admin/settlement-adjustments?merchant_id=${merchantId}&limit=20`, {
+    _label: 'GET /admin/settlement-adjustments'
+  });
+  if (
+    !Array.isArray(settlementAdjustmentReport?.data?.rows) ||
+    settlementAdjustmentReport.data.rows.length < 1 ||
+    Number(settlementAdjustmentReport.data.signed_amount || 0) !== -100 ||
+    !settlementAdjustmentReport.data.rows.some((row) => row.batch_id === settlementBatch.id && Number(row.signed_amount || 0) === -100)
+  ) {
+    throw new Error('settlement adjustment report missing expected row or totals');
+  }
+  const settlementAdjustmentCsv = await req(`/admin/settlement-adjustments/export.csv?merchant_id=${merchantId}&limit=20`, {
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'GET /admin/settlement-adjustments/export.csv'
+  });
+  if (
+    !String(settlementAdjustmentCsv).includes('adjustment_id,batch_id,batch_code') ||
+    !String(settlementAdjustmentCsv).includes('Smoke reconciliation fee correction')
+  ) {
+    throw new Error('settlement adjustment CSV export missing expected header or row');
+  }
+  const merchantLogin = await req('/merchant/login', {
+    method: 'POST',
+    body: { username: merchantId, password: MERCHANT_PORTAL_PASSWORD },
+    _label: 'POST /merchant/login'
+  });
+  if (merchantLogin?.data?.merchant?.id !== merchantId || !merchantLogin?.data?.token) {
+    throw new Error('merchant login did not return expected merchant session');
+  }
+  const merchantSummary = await reqMerchant('/merchant/settlement-summary', merchantId, {
+    _label: 'GET /merchant/settlement-summary'
+  });
+  const merchantPaidAmount = Number(merchantSummary?.data?.paid_amount || 0);
+  const merchantAdjustmentAmount = Number(merchantSummary?.data?.adjustment_amount || 0);
+  const merchantAdjustedPaidAmount = Number(merchantSummary?.data?.adjusted_paid_amount || 0);
+  if (
+    merchantPaidAmount < Number(settlementBatch.net_payable_amount || 0) ||
+    merchantAdjustmentAmount !== -100 ||
+    Math.abs(merchantAdjustedPaidAmount - (merchantPaidAmount - 100)) > 0.01
+  ) {
+    throw new Error('merchant settlement summary missing paid amount');
+  }
+  await reqMerchant('/merchant/settlement-batches', merchantId, {
+    _label: 'GET /merchant/settlement-batches'
+  });
+  const merchantBatchDetail = await reqMerchant(`/merchant/settlement-batches/${settlementBatch.id}`, merchantId, {
+    _label: 'GET /merchant/settlement-batches/:id'
+  });
+  if (!merchantBatchDetail?.data?.events?.some((event) => event.event_type === 'marked_paid')) {
+    throw new Error('merchant settlement batch detail missing payout event history');
+  }
+  if (!Array.isArray(merchantBatchDetail?.data?.adjustments) || merchantBatchDetail.data.adjustments.length < 1) {
+    throw new Error('merchant settlement batch detail missing formal adjustment rows');
+  }
+  const merchantSettlementCsv = await req(`/merchant/settlement-batches/${settlementBatch.id}/export.csv`, {
+    headers: { Authorization: `Bearer ${MERCHANT_TOKEN}`, 'X-Merchant-Id': merchantId },
+    _label: 'GET /merchant/settlement-batches/:id/export.csv'
+  });
+  if (!String(merchantSettlementCsv).includes('batch_code,batch_status,merchant_id')) {
+    throw new Error('merchant settlement CSV export missing expected header');
+  }
+  const merchantSettlementBankCsv = await req(`/merchant/settlement-batches/${settlementBatch.id}/export.csv?format=bank_generic`, {
+    headers: { Authorization: `Bearer ${MERCHANT_TOKEN}`, 'X-Merchant-Id': merchantId },
+    _label: 'GET /merchant/settlement-batches/:id/export.csv bank_generic'
+  });
+  if (!String(merchantSettlementBankCsv).includes('transfer_type,beneficiary_account_reference,beneficiary_account_type')) {
+    throw new Error('merchant settlement bank generic CSV export missing expected header');
+  }
   await reqAdmin(`/admin/audit-logs?entity_id=${txId}`, { _label: 'GET /admin/audit-logs' });
   await reqAdmin(`/admin/transactions/${txId}/heartbeats`, { _label: 'GET /admin/transactions/:id/heartbeats' });
   await reqAdmin(`/admin/devices/${deviceId}/heartbeats`, { _label: 'GET /admin/devices/:id/heartbeats' });
@@ -204,7 +407,20 @@ async function reqDevice(path, opts = {}) {
     body: {},
     _label: 'POST /admin/transactions/:id/retry-notification'
   });
-  await reqAdmin('/admin/dashboard/summary', { _label: 'GET /admin/dashboard/summary' });
+  const dashboardSummary = await reqAdmin('/admin/dashboard/summary', { _label: 'GET /admin/dashboard/summary' });
+  const dashboardData = dashboardSummary?.data || {};
+  const dashboardPaidAmount = Number(dashboardData.settlement_paid_amount || 0);
+  const dashboardAdjustmentAmount = Number(dashboardData.settlement_adjustment_amount || 0);
+  const dashboardAdjustedPaidAmount = Number(dashboardData.settlement_adjusted_paid_amount || 0);
+  if (
+    dashboardPaidAmount < Number(settlementBatch.net_payable_amount || 0) ||
+    dashboardAdjustmentAmount !== -100 ||
+    Math.abs(dashboardAdjustedPaidAmount - (dashboardPaidAmount - 100)) > 0.01 ||
+    Number(dashboardData.settlement_paid_batches || 0) < 1 ||
+    Number(dashboardData.settlement_total_batches || 0) < 1
+  ) {
+    throw new Error('dashboard settlement finance summary missing paid settlement aggregate');
+  }
 
   const noBindingOutlet = await reqAdmin(`/admin/merchants/${merchantId}/outlets`, {
     method: 'POST',
@@ -253,6 +469,129 @@ async function reqDevice(path, opts = {}) {
     _label: 'GET /admin/ledger-entries no-binding'
   });
   await reqAdmin('/admin/notifications/failed', { _label: 'GET /admin/notifications/failed no-binding' });
+  const failedBatchCreate = await reqAdmin('/admin/settlement-batches', {
+    method: 'POST',
+    body: {
+      merchant_id: merchantId,
+      cutoff_at: new Date().toISOString()
+    },
+    _label: 'POST /admin/settlement-batches failed-case'
+  });
+  const failedBatch = failedBatchCreate?.data?.batches?.[0];
+  if (!failedBatch) {
+    throw new Error('failed settlement case did not create a batch');
+  }
+  await reqAdmin(`/admin/settlement-batches/${failedBatch.id}/mark-failed`, {
+    method: 'POST',
+    body: {
+      reason: 'Smoke payout rail rejected transfer',
+      note: 'Smoke failed settlement lifecycle'
+    },
+    _label: 'POST /admin/settlement-batches/:id/mark-failed'
+  });
+  const failedBatchDetail = await reqAdmin(`/admin/settlement-batches/${failedBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id failed'
+  });
+  if (
+    failedBatchDetail?.data?.batch?.status !== 'failed' ||
+    !failedBatchDetail?.data?.events?.some((event) => event.event_type === 'failed')
+  ) {
+    throw new Error('settlement failed lifecycle did not persist failed status/event');
+  }
+  const reprocessedFailed = await reqAdmin(`/admin/settlement-batches/${failedBatch.id}/reprocess`, {
+    method: 'POST',
+    body: {},
+    _label: 'POST /admin/settlement-batches/:id/reprocess failed'
+  });
+  const reprocessedFailedBatch = reprocessedFailed?.data?.new_batch;
+  if (!reprocessedFailedBatch || reprocessedFailedBatch.status !== 'created') {
+    throw new Error('failed settlement reprocess did not create a new created batch');
+  }
+  const reprocessedFailedSource = await reqAdmin(`/admin/settlement-batches/${failedBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id reprocessed-source'
+  });
+  if (!reprocessedFailedSource?.data?.events?.some((event) => event.event_type === 'reprocessed')) {
+    throw new Error('failed settlement reprocess did not create source reprocessed event');
+  }
+  await reqExpect(`/admin/settlement-batches/${failedBatch.id}/reprocess`, 409, {
+    method: 'POST',
+    body: {},
+    headers: { Authorization: `Bearer ${ADMIN_TOKEN}` },
+    _label: 'POST /admin/settlement-batches/:id/reprocess duplicate'
+  });
+
+  const cancelTx = await reqAdmin('/admin/transactions', {
+    method: 'POST',
+    body: {
+      partner_reference: `PR-CANCEL-${ts}`,
+      merchant_id: merchantId,
+      outlet_id: outletId,
+      terminal_id: terminalId,
+      amount: 7700,
+      currency: 'IDR',
+      qr_mode: 'static',
+      initiation_mode: 'static',
+      status: 'initiated'
+    },
+    _label: 'POST /admin/transactions cancel-settlement'
+  });
+  const cancelCallback = {
+    partner_reference: `PR-CANCEL-${ts}`,
+    partner_txn_id: `PTX-CANCEL-${ts}`,
+    amount: 7700,
+    currency: 'IDR',
+    payment_status: 'paid',
+    status: 'paid',
+    paid_at: new Date().toISOString()
+  };
+  const cancelSignature = createHmac('sha256', SECRET).update(JSON.stringify(cancelCallback)).digest('hex');
+  await req('/integrations/qris/callback', {
+    method: 'POST',
+    headers: { 'X-Partner-Signature': cancelSignature },
+    body: { ...cancelCallback, signature: cancelSignature },
+    _label: 'POST /integrations/qris/callback cancel-settlement'
+  });
+  const cancelBatchCreate = await reqAdmin('/admin/settlement-batches', {
+    method: 'POST',
+    body: {
+      merchant_id: merchantId,
+      cutoff_at: new Date().toISOString()
+    },
+    _label: 'POST /admin/settlement-batches cancel-case'
+  });
+  const cancelBatch = cancelBatchCreate?.data?.batches?.[0];
+  if (!cancelBatch) {
+    throw new Error('cancel settlement case did not create a batch');
+  }
+  await reqAdmin(`/admin/settlement-batches/${cancelBatch.id}/cancel`, {
+    method: 'POST',
+    body: {
+      reason: 'Smoke duplicate manual payout batch',
+      note: `cancel tx ${cancelTx?.data?.id}`
+    },
+    _label: 'POST /admin/settlement-batches/:id/cancel'
+  });
+  const cancelBatchDetail = await reqAdmin(`/admin/settlement-batches/${cancelBatch.id}`, {
+    _label: 'GET /admin/settlement-batches/:id cancelled'
+  });
+  if (
+    cancelBatchDetail?.data?.batch?.status !== 'cancelled' ||
+    !cancelBatchDetail?.data?.events?.some((event) => event.event_type === 'cancelled')
+  ) {
+    throw new Error('settlement cancel lifecycle did not persist cancelled status/event');
+  }
+  const reconciliationReport = await reqAdmin('/admin/reconciliation/settlement-batches?limit=50', {
+    _label: 'GET /admin/reconciliation/settlement-batches'
+  });
+  const reconciliationData = reconciliationReport?.data || {};
+  if (
+    !Array.isArray(reconciliationData.rows) ||
+    !Number.isFinite(Number(reconciliationData.total_batches)) ||
+    !Number.isFinite(Number(reconciliationData.mismatch_batches)) ||
+    Number(reconciliationData.total_batches) < 1
+  ) {
+    throw new Error('settlement reconciliation report missing expected aggregate rows');
+  }
 
   const dynamicOutlet = await reqAdmin(`/admin/merchants/${merchantId}/outlets`, {
     method: 'POST',
@@ -303,6 +642,21 @@ async function reqDevice(path, opts = {}) {
     },
     _label: 'POST /device/transactions/dynamic-qr unsupported device'
   });
+  await reqExpect('/device/heartbeat', 403, {
+    method: 'POST',
+    headers: {
+      'X-Device-Id': deviceId,
+      'X-Device-Secret': deviceSecret
+    },
+    body: {
+      device_id: dynamicDeviceId,
+      timestamp: new Date().toISOString(),
+      network_strength: 80,
+      battery_level: 70,
+      state: 'wrong-device'
+    },
+    _label: 'POST /device/heartbeat credential wrong device'
+  });
   const dynamicRequestId = `DYN-REQ-${ts}`;
   const dynamicQr = await reqDevice('/device/transactions/dynamic-qr', {
     method: 'POST',