initial prototype based on nat-as-server

2026-06-29 12:39:55 +02:00
commit f1caa9e681
148 changed files with 34905 additions and 0 deletions
--- a/lib/read_db_cache.php
+++ b/lib/read_db_cache.php
@ -0,0 +1,299 @@
+<?php
+/**
+ * Per-worker read-only DB snapshot in RAM (memfd on Linux) — no plaintext on durable disk.
+ */
+
+/** Passed as $tmpDb when the PDO comes from the worker cache (qdb_discard is a no-op). */
+define('QDB_READONLY_CACHE_HANDLE', "\0qdb_cached_read");
+
+define('QDB_READ_GENERATION_FILE', QDB_UPLOADS_DIR . '/.qdb_read_generation');
+
+/**
+ * @return array{0: PDO, 1: string, 2: null}
+ */
+function qdb_read_db_open(): array {
+    $signature = qdb_read_cache_signature();
+    $cached = qdb_read_cache_get($signature);
+    if ($cached !== null) {
+        return [$cached, QDB_READONLY_CACHE_HANDLE, null];
+    }
+
+    [$pdo, $memfdPath, $memfdFd] = qdb_read_materialize_decrypted_db();
+    // Materialize may persist a migration (updates QDB_PATH mtime); refresh signature before caching.
+    $signature = qdb_read_cache_signature();
+    qdb_read_cache_store($signature, $pdo, $memfdPath, $memfdFd);
+    return [$pdo, QDB_READONLY_CACHE_HANDLE, null];
+}
+
+function qdb_read_cache_invalidate(): void {
+    qdb_read_cache_reset();
+    qdb_read_generation_bump();
+}
+
+function qdb_read_cache_signature(): string {
+    $gen = qdb_read_generation_current();
+    if (!is_file(QDB_PATH)) {
+        return $gen . ':missing';
+    }
+    $mtime = @filemtime(QDB_PATH);
+    $size = @filesize(QDB_PATH);
+    return $gen . ':' . (int)$mtime . ':' . (int)$size;
+}
+
+function qdb_read_generation_current(): string {
+    if (!is_file(QDB_READ_GENERATION_FILE)) {
+        return '0';
+    }
+    $raw = @file_get_contents(QDB_READ_GENERATION_FILE);
+    return $raw === false || $raw === '' ? '0' : trim($raw);
+}
+
+function qdb_read_generation_bump(): void {
+    $dir = dirname(QDB_READ_GENERATION_FILE);
+    if (!is_dir($dir)) {
+        @mkdir($dir, 0750, true);
+    }
+    $next = (string)((int)qdb_read_generation_current() + 1);
+    @file_put_contents(QDB_READ_GENERATION_FILE, $next, LOCK_EX);
+}
+
+/**
+ * @return array{0: PDO, 1: string, 2: int} memfdPath empty when using disk fallback
+ */
+function qdb_read_materialize_decrypted_db(): array {
+    $dbPath = QDB_PATH;
+    $masterKey = get_master_key_bytes();
+    $sql = file_get_contents(QDB_SCHEMA);
+    if ($sql === false) {
+        throw new Exception('Could not read schema.sql');
+    }
+
+    if (!file_exists($dbPath) || !is_file($dbPath)) {
+        $mem = qdb_read_open_sqlite_on_path(':memory:', $sql, true);
+        return [$mem, '', -1];
+    }
+
+    $storedEnc = @file_get_contents($dbPath);
+    if ($storedEnc === false) {
+        throw new Exception('Could not read stored DB');
+    }
+    $decrypted = aes256_cbc_decrypt_bytes($storedEnc, $masterKey);
+
+    $memfd = qdb_memfd_write($decrypted);
+    if ($memfd !== null) {
+        [$fd, $path] = $memfd;
+        try {
+            $pdo = qdb_read_open_sqlite_on_path($path, $sql, false);
+            return [$pdo, $path, $fd];
+        } catch (Throwable $e) {
+            // memfd is filled but SQLite cannot open /proc/self/fd on this host.
+            qdb_close_fd($fd);
+        }
+    }
+
+    $snapshotPath = qdb_read_ram_snapshot_path();
+    if (file_put_contents($snapshotPath, $decrypted) === false) {
+        throw new Exception('Could not write read snapshot');
+    }
+    @chmod($snapshotPath, 0600);
+    $pdo = qdb_read_open_sqlite_on_path($snapshotPath, $sql, false);
+    return [$pdo, $snapshotPath, -1];
+}
+
+/**
+ * RAM-backed snapshot path (tmpfs). One file per worker; overwritten on refresh.
+ */
+function qdb_read_ram_snapshot_path(): string {
+    $dir = is_writable('/dev/shm') ? '/dev/shm' : sys_get_temp_dir();
+    return $dir . '/qdb_read_' . getmypid() . '.sqlite';
+}
+
+/**
+ * @return FFI|null libc handle with memfd_create + close (Linux only)
+ */
+function qdb_linux_libc_ffi() {
+    static $ffi = null;
+    if ($ffi !== null) {
+        return $ffi;
+    }
+    if (PHP_OS_FAMILY !== 'Linux' || !extension_loaded('ffi')) {
+        return null;
+    }
+    try {
+        $ffi = FFI::cdef(
+            'int memfd_create(const char *name, unsigned int flags);
+             long write(int fd, const void *buf, unsigned long count);
+             int close(int fd);',
+            'libc.so.6'
+        );
+    } catch (Throwable $e) {
+        error_log('qdb_linux_libc_ffi: ' . $e->getMessage());
+        return null;
+    }
+    return $ffi;
+}
+
+function qdb_close_fd(int $fd): void {
+    if ($fd < 0) {
+        return;
+    }
+    if (function_exists('posix_close')) {
+        @posix_close($fd);
+        return;
+    }
+    $ffi = qdb_linux_libc_ffi();
+    if ($ffi !== null) {
+        $ffi->close($fd);
+    }
+}
+
+/**
+ * Fill a raw fd (e.g. memfd). /proc/self/fd/N writes are blocked on some hosts;
+ * fall back to libc write() via FFI.
+ */
+function qdb_memfd_fill_fd(int $fd, string $bytes): bool {
+    $len = strlen($bytes);
+    if ($len === 0) {
+        return true;
+    }
+    $written = @file_put_contents('/proc/self/fd/' . $fd, $bytes);
+    if ($written === $len) {
+        return true;
+    }
+    $ffi = qdb_linux_libc_ffi();
+    if ($ffi === null) {
+        return false;
+    }
+    $offset = 0;
+    while ($offset < $len) {
+        $chunkLen = min(1024 * 1024, $len - $offset);
+        $chunk = substr($bytes, $offset, $chunkLen);
+        $buf = FFI::new('char[' . $chunkLen . ']');
+        FFI::memcpy($buf, $chunk, $chunkLen);
+        $n = (int)$ffi->write($fd, $buf, $chunkLen);
+        if ($n <= 0) {
+            return false;
+        }
+        $offset += $n;
+    }
+    return true;
+}
+
+/**
+ * @return array{0: int, 1: string}|null [fd, /proc/self/fd/N]
+ */
+function qdb_memfd_write(string $bytes): ?array {
+    $ffi = qdb_linux_libc_ffi();
+    if ($ffi === null) {
+        return null;
+    }
+    try {
+        $MFD_CLOEXEC = 1;
+        $fd = (int)$ffi->memfd_create('qdb_read', $MFD_CLOEXEC);
+        if ($fd < 0) {
+            return null;
+        }
+        if (!qdb_memfd_fill_fd($fd, $bytes)) {
+            qdb_close_fd($fd);
+            return null;
+        }
+        return [$fd, '/proc/self/fd/' . $fd];
+    } catch (Throwable $e) {
+        error_log('qdb_memfd_write: ' . $e->getMessage());
+        return null;
+    }
+}
+
+function qdb_read_open_sqlite_on_path(string $sqlitePath, string $schemaSql, bool $fresh): PDO {
+    if ($fresh) {
+        $pdo = new PDO('sqlite:' . $sqlitePath);
+        $pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+        $pdo->exec($schemaSql);
+        $pdo->exec('PRAGMA user_version = ' . QDB_VERSION . ';');
+    } else {
+        $pdo = new PDO('sqlite:' . $sqlitePath);
+        $pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+    }
+    $pdo->exec('PRAGMA foreign_keys = ON;');
+    // DELETE (not WAL) so a migration persist via file_get_contents + qdb_save is complete.
+    $pdo->exec('PRAGMA journal_mode = DELETE;');
+
+    $schemaChanged = qdb_apply_migrations($pdo, $schemaSql);
+    if ($schemaChanged) {
+        $migrateLock = qdb_acquire_lock();
+        try {
+            if ($sqlitePath === ':memory:') {
+                throw new Exception('Cannot persist migration from in-memory read snapshot');
+            }
+            $savePath = $sqlitePath;
+            if (str_starts_with($sqlitePath, '/proc/')) {
+                $savePath = tempnam(sys_get_temp_dir(), 'qdb_migrate_');
+                $bytes = file_get_contents($sqlitePath);
+                if ($bytes === false || file_put_contents($savePath, $bytes) === false) {
+                    throw new Exception('Could not copy memfd DB for migration save');
+                }
+            } elseif (!is_file($savePath)) {
+                throw new Exception('Cannot persist migration from missing read snapshot');
+            }
+            $pdo->exec('PRAGMA wal_checkpoint(FULL);');
+            qdb_save($savePath, $migrateLock);
+            if ($savePath !== $sqlitePath && is_file($savePath)) {
+                @unlink($savePath);
+            }
+        } catch (Throwable $e) {
+            flock($migrateLock, LOCK_UN);
+            fclose($migrateLock);
+            throw $e;
+        }
+        return qdb_read_db_open()[0];
+    }
+
+    return $pdo;
+}
+
+/**
+ * @return PDO|null
+ */
+function qdb_read_cache_get(string $signature) {
+    $state = &qdb_read_cache_state();
+    if ($state['signature'] === $signature && $state['pdo'] instanceof PDO) {
+        return $state['pdo'];
+    }
+    return null;
+}
+
+function qdb_read_cache_store(string $signature, PDO $pdo, string $memfdPath, int $memfdFd): void {
+    qdb_read_cache_reset();
+    $state = &qdb_read_cache_state();
+    $state['signature'] = $signature;
+    $state['pdo'] = $pdo;
+    $state['memfdPath'] = $memfdPath;
+    $state['memfdFd'] = $memfdFd;
+}
+
+function qdb_read_cache_reset(): void {
+    $state = &qdb_read_cache_state();
+    $state['signature'] = '';
+    $state['pdo'] = null;
+    if ($state['memfdFd'] >= 0) {
+        qdb_close_fd($state['memfdFd']);
+    }
+    $state['memfdFd'] = -1;
+    if ($state['memfdPath'] !== '' && is_file($state['memfdPath'])) {
+        @unlink($state['memfdPath']);
+    }
+    $state['memfdPath'] = '';
+}
+
+/**
+ * @return array{signature: string, pdo: ?PDO, memfdPath: string, memfdFd: int}
+ */
+function &qdb_read_cache_state(): array {
+    static $state = [
+        'signature' => '',
+        'pdo'       => null,
+        'memfdPath' => '',
+        'memfdFd'     => -1,
+    ];
+    return $state;
+}