Initial commit: Elydora Open Source — self-hosted audit platform for AI agents

Full self-hosted port of the Elydora tamper-evident audit protocol: - Hono API server with PostgreSQL, Redis, MinIO adapters - Ed25519 signed operations, chain hashing, Merkle epoch rollups - PBKDF2 password hashing, HMAC-SHA256 JWT, 5-role RBAC - Next.js management console - Node.js, Python, and Go SDKs - Docker Compose for local deployment - Helm chart for Kubernetes - Comprehensive security hardening and audit fixes applied Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-02 13:41:42 -05:00
1 parent 15547a8c35
commit e390e68594
@@ -0,0 +1,36 @@
+/**
+ * BullMQ queue adapter implementing the MessageQueue interface.
+ *
+ * Wraps a BullMQ Queue instance to replace Cloudflare's Queue producer.
+ * Jobs are enqueued with retry-on-failure semantics (3 attempts,
+ * exponential back-off) matching the Cloudflare Queues default behaviour.
+ */
+
+import { Queue as BullQueue } from 'bullmq';
+import type { ConnectionOptions } from 'bullmq';
+import type { MessageQueue } from './interfaces.js';
+import { generateUUIDv7 } from '../utils/uuid.js';
+
+export const QUEUE_NAME = 'elydora-queue';
+
+export class BullMQAdapter implements MessageQueue {
+  private readonly queue: BullQueue;
+
+  constructor(connection: ConnectionOptions) {
+    this.queue = new BullQueue(QUEUE_NAME, { connection });
+  }
+
+  async send(body: unknown): Promise<{ messageId: string }> {
+    const job = await this.queue.add('message', body as Record<string, unknown>, {
+      attempts: 3,
+      backoff: { type: 'exponential', delay: 1000 },
+      removeOnComplete: { count: 100 },
+      removeOnFail: { count: 500 },
+    });
+    return { messageId: job.id ?? generateUUIDv7() };
+  }
+
+  async close(): Promise<void> {
+    await this.queue.close();
+  }
+}
@@ -0,0 +1,122 @@
+/**
+ * Adapter interfaces for ElydoraOpenSource infrastructure services.
+ *
+ * These interfaces mirror the shape of Cloudflare's D1, R2, KV, and Queue
+ * APIs so that all existing service code can run unchanged against
+ * PostgreSQL, MinIO, Redis, and BullMQ implementations.
+ */
+
+// ---------------------------------------------------------------------------
+// Database (replaces D1Database)
+// ---------------------------------------------------------------------------
+
+/**
+ * A prepared (and optionally bound) SQL statement.
+ * Mirrors D1PreparedStatement's interface exactly.
+ */
+export interface PreparedStatement {
+  /**
+   * Bind positional parameters (? placeholders) to the statement.
+   * Returns a new statement with the values set.
+   */
+  bind(...values: unknown[]): PreparedStatement;
+
+  /** Execute and return the first row, or null if no rows match. */
+  first<T = unknown>(): Promise<T | null>;
+
+  /** Execute and return all matching rows. */
+  all<T = unknown>(): Promise<{ results: T[] }>;
+
+  /** Execute a write statement (INSERT/UPDATE/DELETE). */
+  run(): Promise<{ success: boolean }>;
+}
+
+/**
+ * Relational database adapter.
+ * Mirrors the subset of D1Database methods used by Elydora services.
+ */
+export interface Database {
+  /** Prepare a SQL statement for execution. */
+  prepare(sql: string): PreparedStatement;
+
+  /**
+   * Execute multiple prepared statements atomically (within a transaction).
+   * Mirrors D1Database.batch().
+   */
+  batch(statements: PreparedStatement[]): Promise<{ results: unknown[]; success: boolean }[]>;
+}
+
+// ---------------------------------------------------------------------------
+// Object store (replaces R2Bucket)
+// ---------------------------------------------------------------------------
+
+export interface ObjectStorePutOptions {
+  httpMetadata?: { contentType?: string };
+  customMetadata?: Record<string, string>;
+}
+
+export interface ObjectStoreObject {
+  /** Readable body stream of the stored object. */
+  readonly body: ReadableStream;
+  /** Object size in bytes, if known. */
+  readonly size?: number;
+  /** HTTP metadata associated with the object. */
+  readonly httpMetadata?: { readonly contentType?: string };
+  /** Parse the body as JSON. */
+  json<T = unknown>(): Promise<T>;
+  /** Return the body as a UTF-8 string. */
+  text(): Promise<string>;
+}
+
+export interface ObjectStoreHead {
+  readonly size?: number;
+  readonly httpMetadata?: { readonly contentType?: string };
+}
+
+/**
+ * Object/blob storage adapter.
+ * Mirrors the subset of R2Bucket methods used by Elydora services.
+ */
+export interface ObjectStore {
+  /** Upload an object to the store. */
+  put(
+    key: string,
+    body: string | Uint8Array | ReadableStream,
+    options?: ObjectStorePutOptions,
+  ): Promise<void>;
+
+  /** Retrieve an object, or null if not found. */
+  get(key: string): Promise<ObjectStoreObject | null>;
+
+  /** Check if an object exists (metadata only). Returns null if not found. */
+  head(key: string): Promise<ObjectStoreHead | null>;
+}
+
+// ---------------------------------------------------------------------------
+// Cache (replaces KVNamespace)
+// ---------------------------------------------------------------------------
+
+/**
+ * Key/value cache adapter.
+ * Mirrors the subset of KVNamespace methods used by Elydora services.
+ */
+export interface Cache {
+  /** Retrieve a cached value, or null if not found / expired. */
+  get(key: string): Promise<string | null>;
+
+  /** Store a value in the cache with an optional TTL in seconds. */
+  put(key: string, value: string, options?: { expirationTtl?: number }): Promise<void>;
+}
+
+// ---------------------------------------------------------------------------
+// Message queue (replaces Queue)
+// ---------------------------------------------------------------------------
+
+/**
+ * Message queue adapter.
+ * Mirrors the subset of Cloudflare Queue methods used by Elydora services.
+ */
+export interface MessageQueue {
+  /** Enqueue a message. Returns the assigned message ID. */
+  send(body: unknown): Promise<{ messageId: string }>;
+}
@@ -0,0 +1,151 @@
+/**
+ * MinIO / S3-compatible object storage adapter implementing the ObjectStore interface.
+ *
+ * Uses the AWS SDK v3 S3 client configured with a custom endpoint for MinIO.
+ * Responses are buffered in memory so that both streaming (body) and random
+ * access (json/text) are available on the returned ObjectStoreObject.
+ */
+
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+  type PutObjectCommandInput,
+} from '@aws-sdk/client-s3';
+import { Readable } from 'stream';
+import type {
+  ObjectStore,
+  ObjectStoreObject,
+  ObjectStoreHead,
+  ObjectStorePutOptions,
+} from './interfaces.js';
+
+// ---------------------------------------------------------------------------
+// S3 error detection
+// ---------------------------------------------------------------------------
+
+function isNoSuchKey(err: unknown): boolean {
+  if (!err || typeof err !== 'object') return false;
+  const code = (err as Record<string, unknown>).Code ?? (err as Record<string, unknown>).name;
+  return code === 'NoSuchKey' || code === 'NotFound';
+}
+
+// ---------------------------------------------------------------------------
+// MinIO adapter
+// ---------------------------------------------------------------------------
+
+export class MinioAdapter implements ObjectStore {
+  private readonly client: S3Client;
+  private readonly bucket: string;
+
+  constructor(endpoint: string, accessKey: string, secretKey: string, bucket: string) {
+    this.client = new S3Client({
+      endpoint,
+      region: 'us-east-1', // required by S3 SDK, MinIO ignores it
+      credentials: {
+        accessKeyId: accessKey,
+        secretAccessKey: secretKey,
+      },
+      forcePathStyle: true, // required for MinIO
+    });
+    this.bucket = bucket;
+  }
+
+  async put(
+    key: string,
+    body: string | Uint8Array | ReadableStream,
+    options?: ObjectStorePutOptions,
+  ): Promise<void> {
+    let bodyData: Buffer | string | Uint8Array;
+
+    if (typeof body === 'string') {
+      bodyData = body;
+    } else if (body instanceof Uint8Array) {
+      bodyData = body;
+    } else {
+      // Web ReadableStream — collect into buffer
+      const chunks: Uint8Array[] = [];
+      const reader = body.getReader();
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (value) chunks.push(value);
+      }
+      bodyData = Buffer.concat(chunks);
+    }
+
+    const params: PutObjectCommandInput = {
+      Bucket: this.bucket,
+      Key: key,
+      Body: bodyData,
+    };
+
+    if (options?.httpMetadata?.contentType) {
+      params.ContentType = options.httpMetadata.contentType;
+    }
+
+    if (options?.customMetadata) {
+      params.Metadata = options.customMetadata;
+    }
+
+    await this.client.send(new PutObjectCommand(params));
+  }
+
+  async get(key: string): Promise<ObjectStoreObject | null> {
+    try {
+      const response = await this.client.send(
+        new GetObjectCommand({ Bucket: this.bucket, Key: key }),
+      );
+
+      if (!response.Body) return null;
+
+      // Buffer the entire response so that json(), text(), and body can all work
+      const nodeReadable = response.Body as Readable;
+      const chunks: Buffer[] = [];
+      for await (const chunk of nodeReadable) {
+        chunks.push(chunk as Buffer);
+      }
+      const buffer = Buffer.concat(chunks);
+      const contentType = response.ContentType;
+      const contentLength = response.ContentLength;
+
+      return {
+        get body(): ReadableStream {
+          return new ReadableStream({
+            start(controller) {
+              controller.enqueue(buffer);
+              controller.close();
+            },
+          });
+        },
+        size: contentLength,
+        httpMetadata: contentType ? { contentType } : undefined,
+        async json<T = unknown>(): Promise<T> {
+          return JSON.parse(buffer.toString('utf-8')) as T;
+        },
+        async text(): Promise<string> {
+          return buffer.toString('utf-8');
+        },
+      };
+    } catch (err) {
+      if (isNoSuchKey(err)) return null;
+      throw err;
+    }
+  }
+
+  async head(key: string): Promise<ObjectStoreHead | null> {
+    try {
+      const response = await this.client.send(
+        new HeadObjectCommand({ Bucket: this.bucket, Key: key }),
+      );
+      return {
+        size: response.ContentLength,
+        httpMetadata: response.ContentType ? { contentType: response.ContentType } : undefined,
+      };
+    } catch (err) {
+      if (isNoSuchKey(err)) return null;
+      throw err;
+    }
+  }
+}
@@ -0,0 +1,130 @@
+/**
+ * PostgreSQL adapter implementing the Database interface.
+ *
+ * Translates the D1-style API (prepare/bind/first/all/run/batch) into
+ * standard `pg` Pool queries. SQLite `?` placeholders are converted to
+ * PostgreSQL `$N` positional parameters automatically.
+ *
+ * Batch execution runs all statements inside a single transaction to
+ * preserve the atomicity guarantee of D1.batch().
+ */
+
+import pg from 'pg';
+import type { Database, PreparedStatement } from './interfaces.js';
+
+const { Pool, types } = pg;
+type Pool = InstanceType<typeof pg.Pool>;
+type PoolClient = pg.PoolClient;
+
+// Parse PostgreSQL BIGINT (oid 20) as JavaScript number instead of string.
+// Elydora timestamps are Unix milliseconds (~1.7e12) which fit safely within
+// Number.MAX_SAFE_INTEGER (~9e15).
+types.setTypeParser(20, (val: string) => Number(val));
+
+// ---------------------------------------------------------------------------
+// Placeholder conversion
+// ---------------------------------------------------------------------------
+
+/**
+ * Convert SQLite `?` positional placeholders to PostgreSQL `$N` style.
+ * Only replaces `?` outside of SQL string literals (safe for all Elydora queries).
+ */
+function convertPlaceholders(sql: string): string {
+  let index = 0;
+  return sql.replace(/\?/g, () => `$${++index}`);
+}
+
+// ---------------------------------------------------------------------------
+// Statement implementation
+// ---------------------------------------------------------------------------
+
+/**
+ * Internal PostgreSQL statement that implements PreparedStatement.
+ * Stores the SQL and bound values; executes lazily on first/all/run/executeWith.
+ */
+class PostgresStatement implements PreparedStatement {
+  constructor(
+    private readonly pool: Pool,
+    private readonly sql: string,
+    private readonly values: unknown[],
+  ) {}
+
+  bind(...values: unknown[]): PreparedStatement {
+    return new PostgresStatement(this.pool, this.sql, values);
+  }
+
+  async first<T = unknown>(): Promise<T | null> {
+    const result = await this.pool.query<Record<string, unknown>>({
+      text: convertPlaceholders(this.sql),
+      values: this.values as unknown[],
+    });
+    return (result.rows[0] as T) ?? null;
+  }
+
+  async all<T = unknown>(): Promise<{ results: T[] }> {
+    const result = await this.pool.query<Record<string, unknown>>({
+      text: convertPlaceholders(this.sql),
+      values: this.values as unknown[],
+    });
+    return { results: result.rows as T[] };
+  }
+
+  async run(): Promise<{ success: boolean }> {
+    await this.pool.query({
+      text: convertPlaceholders(this.sql),
+      values: this.values as unknown[],
+    });
+    return { success: true };
+  }
+
+  /** Execute using a specific pooled client (used by batch transaction). */
+  async executeWith(client: PoolClient): Promise<{ results: unknown[]; success: boolean }> {
+    const result = await client.query({
+      text: convertPlaceholders(this.sql),
+      values: this.values as unknown[],
+    });
+    return { results: result.rows, success: true };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// PostgreSQL adapter
+// ---------------------------------------------------------------------------
+
+export class PostgresAdapter implements Database {
+  private readonly pool: Pool;
+
+  constructor(connectionString: string) {
+    this.pool = new Pool({ connectionString });
+  }
+
+  prepare(sql: string): PreparedStatement {
+    return new PostgresStatement(this.pool, sql, []);
+  }
+
+  async batch(
+    statements: PreparedStatement[],
+  ): Promise<{ results: unknown[]; success: boolean }[]> {
+    const client = await this.pool.connect();
+    try {
+      await client.query('BEGIN');
+      const results: { results: unknown[]; success: boolean }[] = [];
+      for (const stmt of statements) {
+        const pgStmt = stmt as PostgresStatement;
+        results.push(await pgStmt.executeWith(client));
+      }
+      await client.query('COMMIT');
+      return results;
+    } catch (err) {
+      await client.query('ROLLBACK');
+      throw err;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Close the connection pool (call on graceful shutdown). */
+  async close(): Promise<void> {
+    await this.pool.end();
+  }
+}
@@ -0,0 +1,41 @@
+/**
+ * Redis cache adapter implementing the Cache interface.
+ *
+ * Uses ioredis to back the nonce-deduplication and chain-hash caching
+ * that was previously served by Cloudflare KVNamespace.
+ */
+
+import { Redis } from 'ioredis';
+import type { Cache } from './interfaces.js';
+
+export class RedisCacheAdapter implements Cache {
+  private readonly client: Redis;
+
+  constructor(redisUrl: string) {
+    this.client = new Redis(redisUrl, { lazyConnect: true });
+  }
+
+  async get(key: string): Promise<string | null> {
+    return this.client.get(key);
+  }
+
+  async put(
+    key: string,
+    value: string,
+    options?: { expirationTtl?: number },
+  ): Promise<void> {
+    if (options?.expirationTtl) {
+      await this.client.set(key, value, 'EX', options.expirationTtl);
+    } else {
+      await this.client.set(key, value);
+    }
+  }
+
+  async connect(): Promise<void> {
+    await this.client.connect();
+  }
+
+  async close(): Promise<void> {
+    await this.client.quit();
+  }
+}