From 86ca518c6cacbf4e87d1c7952a38b4c5d1994f70 Mon Sep 17 00:00:00 2001 From: bach Date: Mon, 16 Nov 2026 20:16:37 +0100 Subject: [PATCH] materio_ai: embedding table and storage service (V3) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM --- .../custom/materio_ai/materio_ai.install | 60 ++++++++ .../custom/materio_ai/materio_ai.services.yml | 5 + .../src/Storage/EmbeddingStorage.php | 137 ++++++++++++++++++ 3 files changed, 202 insertions(+) create mode 100644 web/modules/custom/materio_ai/materio_ai.install create mode 100644 web/modules/custom/materio_ai/src/Storage/EmbeddingStorage.php diff --git a/web/modules/custom/materio_ai/materio_ai.install b/web/modules/custom/materio_ai/materio_ai.install new file mode 100644 index 00000000..51fc6b0f --- /dev/null +++ b/web/modules/custom/materio_ai/materio_ai.install @@ -0,0 +1,60 @@ + 'Vector of each node translation, computed by the embedding model.', + 'fields' => [ + 'nid' => [ + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'description' => 'Node ID.', + ], + 'langcode' => [ + 'type' => 'varchar_ascii', + 'length' => 12, + 'not null' => TRUE, + 'description' => 'Language of the vectorized translation.', + ], + 'model' => [ + 'type' => 'varchar_ascii', + 'length' => 128, + 'not null' => TRUE, + 'description' => 'Model that computed the vector, for example bge-m3:567m.', + ], + 'text_hash' => [ + 'type' => 'varchar_ascii', + 'length' => 64, + 'not null' => TRUE, + 'description' => 'SHA-256 of the vectorized text, to skip unchanged texts.', + ], + 'vector' => [ + 'type' => 'blob', + 'size' => 'normal', + 'not null' => TRUE, + 'description' => 'Vector as little-endian 32-bit floats.', + ], + 'changed' => [ + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => 'Unix timestamp of the last vectorization.', + ], + ], + 'primary key' => ['nid', 'langcode'], + 'indexes' => [ + 'model' => ['model'], + ], + ]; + return $schema; +} diff --git a/web/modules/custom/materio_ai/materio_ai.services.yml b/web/modules/custom/materio_ai/materio_ai.services.yml index 397ce80d..5a713b42 100644 --- a/web/modules/custom/materio_ai/materio_ai.services.yml +++ b/web/modules/custom/materio_ai/materio_ai.services.yml @@ -7,3 +7,8 @@ services: class: Drupal\materio_ai\Embedding\OllamaEmbeddingClient arguments: ['@http_client', '@config.factory'] Drupal\materio_ai\Embedding\EmbeddingClientInterface: '@materio_ai.embedding_client' + + materio_ai.embedding_storage: + class: Drupal\materio_ai\Storage\EmbeddingStorage + arguments: ['@database', '@datetime.time'] + Drupal\materio_ai\Storage\EmbeddingStorage: '@materio_ai.embedding_storage' diff --git a/web/modules/custom/materio_ai/src/Storage/EmbeddingStorage.php b/web/modules/custom/materio_ai/src/Storage/EmbeddingStorage.php new file mode 100644 index 00000000..de2cd310 --- /dev/null +++ b/web/modules/custom/materio_ai/src/Storage/EmbeddingStorage.php @@ -0,0 +1,137 @@ +database->merge(self::TABLE) + ->keys(['nid' => $nid, 'langcode' => $langcode]) + ->fields([ + 'model' => $model, + 'text_hash' => $text_hash, + 'vector' => self::pack($vector), + 'changed' => $this->time->getRequestTime(), + ]) + ->execute(); + } + + /** + * Returns the stored row of a node translation, with the vector unpacked. + * + * @return array{nid: int, langcode: string, model: string, text_hash: string, vector: float[], changed: int}|null + * The row, or NULL when the translation has no vector. + */ + public function load(int $nid, string $langcode): ?array { + $row = $this->database->select(self::TABLE, 'e') + ->fields('e') + ->condition('nid', $nid) + ->condition('langcode', $langcode) + ->execute() + ->fetchAssoc(); + return $row ? self::row($row) : NULL; + } + + /** + * Returns the text hashes of a node, keyed by langcode. + * + * Lets the queue skip texts that did not change, without loading vectors. + * + * @return array + * The hashes and models. + */ + public function loadHashes(int $nid): array { + return $this->database->select(self::TABLE, 'e') + ->fields('e', ['langcode', 'model', 'text_hash']) + ->condition('nid', $nid) + ->execute() + ->fetchAllAssoc('langcode', \PDO::FETCH_ASSOC); + } + + /** + * Deletes the vectors of a node, or of one of its translations. + */ + public function delete(int $nid, ?string $langcode = NULL): int { + $query = $this->database->delete(self::TABLE)->condition('nid', $nid); + if ($langcode !== NULL) { + $query->condition('langcode', $langcode); + } + return (int) $query->execute(); + } + + /** + * Counts the stored vectors, optionally for one model. + */ + public function count(?string $model = NULL): int { + $query = $this->database->select(self::TABLE, 'e'); + if ($model !== NULL) { + $query->condition('model', $model); + } + return (int) $query->countQuery()->execute()->fetchField(); + } + + /** + * Hash of a text, as stored in text_hash. + */ + public static function hash(string $text): string { + return hash('sha256', $text); + } + + /** + * Packs a vector as little-endian 32-bit floats. + * + * @param float[] $vector + * The vector. + */ + public static function pack(array $vector): string { + return pack('g*', ...array_values($vector)); + } + + /** + * Unpacks a vector stored by pack(). + * + * @return float[] + * The vector, keys starting at 0. + */ + public static function unpack(string $blob): array { + return array_values(unpack('g*', $blob)); + } + + /** + * Casts a database row and unpacks its vector. + */ + private static function row(array $row): array { + return [ + 'nid' => (int) $row['nid'], + 'langcode' => $row['langcode'], + 'model' => $row['model'], + 'text_hash' => $row['text_hash'], + 'vector' => self::unpack($row['vector']), + 'changed' => (int) $row['changed'], + ]; + } + +}