server: commit operations before enqueueing and answer seq_no races with PREV_HASH_MISMATCH

The operation and receipt now commit before the queue send. A Postgres
23505 on operations_agent_id_seq_no_key is answered with 400
PREV_HASH_MISMATCH carrying the stored chain hash so hook retries can
resync. A failed enqueue after the commit is logged as
operation.enqueue_failed and the receipt is still returned.

Every producer now names its message: MessageQueue.send takes the
durable message id, and the BullMQ adapter passes it as the job id, so
the receipt's queue_message_id is the real queue identifier.

Claude-Session: https://claude.ai/code/session_01QoUjdc7HGKTsgj5GhG8HvF

packages/server/src/adapters/bullmq.ts

@@ -9,7 +9,6 @@
 import { Queue as BullQueue } from 'bullmq';
 import type { ConnectionOptions } from 'bullmq';
 import type { MessageQueue } from './interfaces.js';
-import { generateUUIDv7 } from '../utils/uuid.js';
 
 export const QUEUE_NAME = 'elydora-queue';
 
@@ -20,14 +19,14 @@ export class BullMQAdapter implements MessageQueue {
     this.queue = new BullQueue(QUEUE_NAME, { connection });
   }
 
-  async send(body: unknown): Promise<{ messageId: string }> {
-    const job = await this.queue.add('message', body as Record<string, unknown>, {
+  async send(messageId: string, body: unknown): Promise<void> {
+    await this.queue.add('message', body as Record<string, unknown>, {
+      jobId: messageId,
       attempts: 3,
       backoff: { type: 'exponential', delay: 1000 },
       removeOnComplete: { count: 100 },
       removeOnFail: { count: 500 },
     });
-    return { messageId: job.id ?? generateUUIDv7() };
   }
 
   async close(): Promise<void> {

packages/server/src/adapters/interfaces.ts

@@ -117,6 +117,6 @@ export interface Cache {
  * Mirrors the subset of Cloudflare Queue methods used by Elydora services.
  */
 export interface MessageQueue {
-  /** Enqueue a message. Returns the assigned message ID. */
-  send(body: unknown): Promise<{ messageId: string }>;
+  /** Enqueue a message under the caller's durable message ID. */
+  send(messageId: string, body: unknown): Promise<void>;
 }

packages/server/src/scheduler.ts

@@ -12,6 +12,7 @@ import { RedisCacheAdapter } from './adapters/redis-cache.js';
 import { BullMQAdapter } from './adapters/bullmq.js';
 import { createEpoch } from './services/epoch-service.js';
 import { DEFAULT_EPOCH_INTERVAL_MS } from './shared/index.js';
+import { generateUUIDv7 } from './utils/uuid.js';
 
 // ---------------------------------------------------------------------------
 // Initialize adapters
@@ -65,7 +66,7 @@ async function runEpochCreation(): Promise<void> {
 
       if (epoch) {
         try {
-          await queue.send({
+          await queue.send(generateUUIDv7(), {
             type: 'tsa',
             epoch_id: epoch.epoch_id,
             org_id: epoch.org_id,

packages/server/src/services/export-service.ts

@@ -99,7 +99,7 @@ export async function createExport(
     .run();
 
   // Enqueue the export job
-  await queue.send({
+  await queue.send(generateUUIDv7(), {
     type: 'export',
     export_id: exportId,
     org_id: orgId,

packages/server/src/services/operation-service.ts

@@ -201,20 +201,10 @@ export async function submitOperation(
   });
 
   // ------------------------------------------------------------------
-  // Step 10: Enqueue for async processing
+  // Step 10: Reserve the queue message ID; the send follows the commit
   // ------------------------------------------------------------------
-  const queueMessageResult = await queue.send({
-    type: 'operation',
-    operation_id: eor.operation_id,
-    org_id: eor.org_id,
-    agent_id: eor.agent_id,
-  });
-
-  // D1 batch insert: operation + receipt
+  const queueMessageId = generateUUIDv7();
   const receiptId = generateUUIDv7();
-  const queueMessageId = typeof queueMessageResult === 'object' && queueMessageResult !== null
-    ? String((queueMessageResult as Record<string, unknown>).messageId ?? receiptId)
-    : receiptId;
 
   // ------------------------------------------------------------------
   // Step 11: Generate EAR receipt
@@ -291,11 +281,45 @@ export async function submitOperation(
       .bind(receiptId, eor.operation_id, r2ReceiptKey, receivedAt),
   );
 
-  await db.batch(statements);
+  try {
+    await db.batch(statements);
+  } catch (error) {
+    if (!isSeqNoConflict(error)) throw error;
+    const winner = await db
+      .prepare('SELECT chain_hash FROM operations WHERE agent_id = ? ORDER BY seq_no DESC LIMIT 1')
+      .bind(eor.agent_id)
+      .first<{ chain_hash: string }>();
+    if (!winner) throw new Error('seq_no conflict without a stored operation.');
+    throw new AppError(400, 'PREV_HASH_MISMATCH', {
+      key: 'operation.prevHashMismatch',
+      params: { expected: winner.chain_hash, actual: eor.prev_chain_hash },
+    });
+  }
+
+  try {
+    await queue.send(queueMessageId, {
+      type: 'operation',
+      operation_id: eor.operation_id,
+      org_id: eor.org_id,
+      agent_id: eor.agent_id,
+    });
+  } catch (error) {
+    console.error('operation.enqueue_failed', {
+      operation_id: eor.operation_id,
+      queue_message_id: queueMessageId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
 
   return { receipt: ear };
 }
 
+function isSeqNoConflict(error: unknown): boolean {
+  if (typeof error !== 'object' || error === null) return false;
+  const { code, constraint } = error as { code?: unknown; constraint?: unknown };
+  return code === '23505' && constraint === 'operations_agent_id_seq_no_key';
+}
+
 // ---------------------------------------------------------------------------
 // Get operation
 // ---------------------------------------------------------------------------