tomh104/nat-as-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

migration to mysql
Some checks failed
PHPUnit / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Tom Hempel
2026-07-10 09:57:44 +02:00
parent 351af170a0
commit 810ed24792
24 changed files with 986 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
db_init.php
View File

@ -1,7 +1,7 @@
<?php
// /var/www/html/db_init.php
// Shared helpers for opening, creating, and saving the encrypted SQLite database.
// Shared helpers for opening, creating, and saving the questionnaire database.
require_once __DIR__ . '/common.php';
require_once __DIR__ . '/lib/sql_dialect.php';
if (defined('QDB_TEST_UPLOADS')) {
define('QDB_UPLOADS_DIR', QDB_TEST_UPLOADS);
@ -15,26 +15,8 @@ if (defined('QDB_TEST_UPLOADS')) {
define('QDB_SCHEMA', __DIR__ . '/schema.sql');
define('QDB_VERSION', 14);
function qdb_table_exists(PDO $pdo, string $table): bool {
$stmt = $pdo->prepare(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = :t LIMIT 1"
);
$stmt->execute([':t' => $table]);
return (bool)$stmt->fetchColumn();
}
function qdb_column_exists(PDO $pdo, string $table, string $column): bool {
$stmt = $pdo->query('PRAGMA table_info(' . $table . ')');
foreach ($stmt->fetchAll(PDO::FETCH_ASSOC) as $col) {
if (($col['name'] ?? '') === $column) {
return true;
}
}
return false;
}
/**
* Apply incremental schema fixes. Returns true if the temp DB was modified.
* Apply incremental schema fixes. Returns true if the DB was modified.
*/
function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
$changed = false;
@ -138,8 +120,8 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
UNIQUE(clientCode, questionnaireID, version)
)
");
$pdo->exec('CREATE INDEX IF NOT EXISTS idx_submission_client_qn ON questionnaire_submission(clientCode, questionnaireID)');
$pdo->exec('CREATE INDEX IF NOT EXISTS idx_submission_client_time ON questionnaire_submission(clientCode, submittedAt)');
qdb_create_index_if_not_exists($pdo, 'idx_submission_client_qn', 'questionnaire_submission', 'clientCode, questionnaireID');
qdb_create_index_if_not_exists($pdo, 'idx_submission_client_time', 'questionnaire_submission', 'clientCode, submittedAt');
$changed = true;
}
@ -211,7 +193,7 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
UNIQUE(questionID, scopeKey, levelKey)
)
");
$pdo->exec('CREATE INDEX IF NOT EXISTS idx_score_rule_question ON question_score_rule(questionID)');
qdb_create_index_if_not_exists($pdo, 'idx_score_rule_question', 'question_score_rule', 'questionID');
$changed = true;
}
@ -267,7 +249,7 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
FOREIGN KEY(profileID) REFERENCES scoring_profile(profileID) ON DELETE CASCADE
)
");
$pdo->exec('CREATE INDEX IF NOT EXISTS idx_client_profile_result_profile ON client_scoring_profile_result(profileID)');
qdb_create_index_if_not_exists($pdo, 'idx_client_profile_result_profile', 'client_scoring_profile_result', 'profileID');
$changed = true;
}
@ -293,7 +275,7 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
}
}
$currentVersion = (int)$pdo->query('PRAGMA user_version')->fetchColumn();
$currentVersion = qdb_schema_version($pdo);
if ($currentVersion < 10
&& qdb_table_exists($pdo, 'completed_questionnaire')
&& qdb_table_exists($pdo, 'client_scoring_profile_result')) {
@ -303,7 +285,7 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
}
}
$currentVersion = (int)$pdo->query('PRAGMA user_version')->fetchColumn();
$currentVersion = qdb_schema_version($pdo);
if ($currentVersion < 7 && qdb_table_exists($pdo, 'question_score_rule')) {
require_once __DIR__ . '/lib/scoring.php';
if (qdb_backfill_glass_score_rules($pdo) > 0) {
@ -317,7 +299,7 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
}
}
$currentVersion = (int)$pdo->query('PRAGMA user_version')->fetchColumn();
$currentVersion = qdb_schema_version($pdo);
if ($currentVersion < 8 && qdb_table_exists($pdo, 'scoring_profile')) {
if (!qdb_column_exists($pdo, 'scoring_profile', 'greenMin')) {
$pdo->exec('ALTER TABLE scoring_profile ADD COLUMN greenMin INTEGER NOT NULL DEFAULT 0');
@ -340,7 +322,7 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
$changed = true;
}
$currentVersion = (int)$pdo->query('PRAGMA user_version')->fetchColumn();
$currentVersion = qdb_schema_version($pdo);
if ($currentVersion < 13 && qdb_table_exists($pdo, 'scoring_profile')) {
if (!qdb_column_exists($pdo, 'scoring_profile', 'processCompleteBands')) {
$pdo->exec("ALTER TABLE scoring_profile ADD COLUMN processCompleteBands TEXT NOT NULL DEFAULT '[]'");
@ -348,7 +330,7 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
}
}
$currentVersion = (int)$pdo->query('PRAGMA user_version')->fetchColumn();
$currentVersion = qdb_schema_version($pdo);
if ($currentVersion < 14) {
if (qdb_table_exists($pdo, 'language') && !qdb_column_exists($pdo, 'language', 'enabled')) {
$pdo->exec('ALTER TABLE language ADD COLUMN enabled INTEGER NOT NULL DEFAULT 1');
@ -368,7 +350,7 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
}
}
$currentVersion = (int)$pdo->query('PRAGMA user_version')->fetchColumn();
$currentVersion = qdb_schema_version($pdo);
if ($currentVersion < 6 && qdb_table_exists($pdo, 'questionnaire_submission')) {
$pdo->exec(
'UPDATE questionnaire_submission SET submittedAt = completedAt
@ -377,27 +359,55 @@ function qdb_apply_migrations(PDO $pdo, string $schemaSql): bool {
$changed = true;
}
$currentVersion = (int)$pdo->query('PRAGMA user_version')->fetchColumn();
$currentVersion = qdb_schema_version($pdo);
if ($currentVersion < QDB_VERSION) {
$pdo->exec('PRAGMA foreign_keys = OFF;');
$pdo->exec($schemaSql);
if (!qdb_column_exists($pdo, 'questionnaire', 'categoryKey')) {
$pdo->exec("ALTER TABLE questionnaire ADD COLUMN categoryKey TEXT NOT NULL DEFAULT ''");
qdb_set_foreign_keys($pdo, false);
try {
qdb_exec_schema_sql($pdo, $schemaSql);
if (!qdb_column_exists($pdo, 'questionnaire', 'categoryKey')) {
$pdo->exec("ALTER TABLE questionnaire ADD COLUMN categoryKey TEXT NOT NULL DEFAULT ''");
}
qdb_set_schema_version($pdo, QDB_VERSION);
} finally {
qdb_set_foreign_keys($pdo, true);
}
$pdo->exec('PRAGMA user_version = ' . QDB_VERSION . ';');
$pdo->exec('PRAGMA foreign_keys = ON;');
$changed = true;
}
return $changed;
}
/**
* Exclusive lock with short retries (non-blocking attempts). Avoids hung PHP workers
* when two writes overlap (app upload + website edit).
*/
function qdb_flock_exclusive_with_retry($lockFp, int $maxAttempts = 12, int $sleepMicros = 75000): void
{
function qdb_mysql_ensure_ready(PDO $pdo): void {
static $ready = false;
if ($ready) {
return;
}
$lockName = 'nat_as_schema_migration';
$lockStmt = $pdo->prepare('SELECT GET_LOCK(:name, 30)');
$lockStmt->execute([':name' => $lockName]);
if ((int)$lockStmt->fetchColumn() !== 1) {
throw new RuntimeException('Could not acquire MySQL schema migration lock');
}
try {
$schemaPath = qdb_schema_file();
$schemaSql = file_get_contents($schemaPath);
if ($schemaSql === false) {
throw new Exception('Could not read MySQL schema');
}
if (!qdb_table_exists($pdo, 'users') || qdb_schema_version($pdo) < QDB_VERSION) {
qdb_exec_schema_file($pdo, $schemaPath);
}
qdb_apply_migrations($pdo, $schemaSql);
$ready = true;
} finally {
$releaseStmt = $pdo->prepare('SELECT RELEASE_LOCK(:name)');
$releaseStmt->execute([':name' => $lockName]);
}
}
function qdb_flock_exclusive_with_retry($lockFp, int $maxAttempts = 12, int $sleepMicros = 75000): void {
for ($attempt = 0; $attempt < $maxAttempts; $attempt++) {
if (flock($lockFp, LOCK_EX | LOCK_NB)) {
return;
@ -409,8 +419,7 @@ function qdb_flock_exclusive_with_retry($lockFp, int $maxAttempts = 12, int $sle
throw new Exception('Could not acquire lock');
}
function qdb_open_writable_lock()
{
function qdb_open_writable_lock() {
$lockFp = fopen(QDB_LOCK, 'c');
if ($lockFp === false) {
throw new Exception('Could not open lock file');
@ -424,14 +433,15 @@ function qdb_acquire_lock() {
}
/**
* Decrypt the master DB file into a temp SQLite file, or create a fresh one
* from schema.sql if no DB exists yet.
*
* Returns [PDO $pdo, string $tmpPath, resource|null $lockFp].
* If $writable is true, an exclusive lock is held -- caller MUST call
* qdb_save() or qdb_discard() afterwards.
*/
function qdb_open(bool $writable = false): array {
if (qdb_is_mysql()) {
[$pdo] = qdb_mysql_open();
qdb_mysql_ensure_ready($pdo);
return [$pdo, '', null];
}
if (!$writable) {
require_once __DIR__ . '/lib/read_db_cache.php';
return qdb_read_db_open();
@ -441,29 +451,28 @@ function qdb_open(bool $writable = false): array {
if (!is_dir(dirname($dbPath))) {
if (!mkdir(dirname($dbPath), 0755, true) && !is_dir(dirname($dbPath))) {
throw new Exception("Could not create uploads directory");
throw new Exception('Could not create uploads directory');
}
}
$lockFp = null;
if ($writable) {
$lockFp = qdb_open_writable_lock();
}
$lockFp = qdb_open_writable_lock();
$tmpDb = tempnam(sys_get_temp_dir(), 'qdb_');
$masterKey = get_master_key_bytes();
$sql = file_get_contents(QDB_SCHEMA);
if ($sql === false) {
if ($lockFp) { flock($lockFp, LOCK_UN); fclose($lockFp); }
throw new Exception("Could not read schema.sql");
flock($lockFp, LOCK_UN);
fclose($lockFp);
throw new Exception('Could not read schema.sql');
}
if (file_exists($dbPath) && is_file($dbPath)) {
$storedEnc = @file_get_contents($dbPath);
if ($storedEnc === false) {
if ($lockFp) { flock($lockFp, LOCK_UN); fclose($lockFp); }
throw new Exception("Could not read stored DB");
flock($lockFp, LOCK_UN);
fclose($lockFp);
throw new Exception('Could not read stored DB');
}
try {
$decrypted = aes256_cbc_decrypt_bytes($storedEnc, $masterKey);
@ -481,45 +490,32 @@ function qdb_open(bool $writable = false): array {
throw $e;
}
if (file_put_contents($tmpDb, $decrypted) === false) {
if ($lockFp) { flock($lockFp, LOCK_UN); fclose($lockFp); }
throw new Exception("Could not write temp DB");
flock($lockFp, LOCK_UN);
fclose($lockFp);
throw new Exception('Could not write temp DB');
}
} else {
$pdo = new PDO("sqlite:$tmpDb");
$pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$pdo->exec($sql);
$pdo->exec("PRAGMA user_version = " . QDB_VERSION . ";");
qdb_set_schema_version($pdo, QDB_VERSION);
$pdo = null;
}
$pdo = new PDO("sqlite:$tmpDb");
$pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$pdo->exec("PRAGMA foreign_keys = ON;");
qdb_set_foreign_keys($pdo, true);
$schemaChanged = qdb_apply_migrations($pdo, $sql);
// Read-only opens discard the temp file; persist one-off migrations to the master DB.
if ($schemaChanged && !$writable) {
$migrateLock = qdb_acquire_lock();
try {
qdb_save($tmpDb, $migrateLock);
} catch (Throwable $e) {
@unlink($tmpDb);
flock($migrateLock, LOCK_UN);
fclose($migrateLock);
throw $e;
}
return qdb_open(false);
}
qdb_apply_migrations($pdo, $sql);
return [$pdo, $tmpDb, $lockFp];
}
/**
* Encrypt the temp DB and atomically write it back to the master path.
* Releases the lock.
*/
function qdb_save(string $tmpDb, $lockFp): void {
if (qdb_is_mysql()) {
return;
}
$masterKey = get_master_key_bytes();
try {
$ck = new PDO('sqlite:' . $tmpDb);
@ -528,17 +524,19 @@ function qdb_save(string $tmpDb, $lockFp): void {
// best-effort before reading plaintext bytes
}
$plainDb = file_get_contents($tmpDb);
if ($plainDb === false) throw new Exception("Could not read temp DB for save");
if ($plainDb === false) {
throw new Exception('Could not read temp DB for save');
}
$enc = aes256_cbc_encrypt_bytes($plainDb, $masterKey);
$dbPath = QDB_PATH;
$tmpEncrypted = tempnam(dirname($dbPath), 'enc_qdb_');
if (file_put_contents($tmpEncrypted, $enc) === false) {
throw new Exception("Could not write encrypted DB");
throw new Exception('Could not write encrypted DB');
}
if (!@rename($tmpEncrypted, $dbPath)) {
if (!@copy($tmpEncrypted, $dbPath) || !@unlink($tmpEncrypted)) {
throw new Exception("Could not save encrypted DB (rename/copy failed)");
throw new Exception('Could not save encrypted DB (rename/copy failed)');
}
}
@chmod($dbPath, 0644);
@ -553,10 +551,10 @@ function qdb_save(string $tmpDb, $lockFp): void {
qdb_read_cache_invalidate();
}
/**
* Discard changes -- clean up temp file and release lock without saving.
*/
function qdb_discard(string $tmpDb, $lockFp): void {
if (qdb_is_mysql()) {
return;
}
if (defined('QDB_READONLY_CACHE_HANDLE') && $tmpDb === QDB_READONLY_CACHE_HANDLE) {
return;
}
@ -567,14 +565,7 @@ function qdb_discard(string $tmpDb, $lockFp): void {
}
}
/**
* Open a read-only DB connection; on failure logs and returns a JSON error response.
*
* @return array{0: PDO, 1: string, 2: resource|null}|null
*/
/**
* @return array{0: PDO, 1: string, 2: resource|null}
*/
/** @return array{0: PDO, 1: string, 2: resource|null} */
function qdb_open_read_or_fail(): array {
try {
return qdb_open(false);
@ -584,9 +575,7 @@ function qdb_open_read_or_fail(): array {
}
}
/**
* @return array{0: PDO, 1: string, 2: resource|null}
*/
/** @return array{0: PDO, 1: string, 2: resource|null} */
function qdb_open_write_or_fail(): array {
try {
return qdb_open(true);