materio_ai: embedding table and storage service (V3)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
2026-11-16 20:16:37 +01:00
co-authored by Claude Opus 5.5
parent 45f21b2e49
commit 86ca518c6c
3 changed files with 202 additions and 0 deletions
@@ -0,0 +1,60 @@
<?php
/**
* @file
* Install, update and uninstall functions for materio_ai.
*/
declare(strict_types=1);
/**
* Implements hook_schema().
*/
function materio_ai_schema(): array {
$schema['materio_ai_embedding'] = [
'description' => 'Vector of each node translation, computed by the embedding model.',
'fields' => [
'nid' => [
'type' => 'int',
'unsigned' => TRUE,
'not null' => TRUE,
'description' => 'Node ID.',
],
'langcode' => [
'type' => 'varchar_ascii',
'length' => 12,
'not null' => TRUE,
'description' => 'Language of the vectorized translation.',
],
'model' => [
'type' => 'varchar_ascii',
'length' => 128,
'not null' => TRUE,
'description' => 'Model that computed the vector, for example bge-m3:567m.',
],
'text_hash' => [
'type' => 'varchar_ascii',
'length' => 64,
'not null' => TRUE,
'description' => 'SHA-256 of the vectorized text, to skip unchanged texts.',
],
'vector' => [
'type' => 'blob',
'size' => 'normal',
'not null' => TRUE,
'description' => 'Vector as little-endian 32-bit floats.',
],
'changed' => [
'type' => 'int',
'not null' => TRUE,
'default' => 0,
'description' => 'Unix timestamp of the last vectorization.',
],
],
'primary key' => ['nid', 'langcode'],
'indexes' => [
'model' => ['model'],
],
];
return $schema;
}
@@ -7,3 +7,8 @@ services:
class: Drupal\materio_ai\Embedding\OllamaEmbeddingClient
arguments: ['@http_client', '@config.factory']
Drupal\materio_ai\Embedding\EmbeddingClientInterface: '@materio_ai.embedding_client'
materio_ai.embedding_storage:
class: Drupal\materio_ai\Storage\EmbeddingStorage
arguments: ['@database', '@datetime.time']
Drupal\materio_ai\Storage\EmbeddingStorage: '@materio_ai.embedding_storage'
@@ -0,0 +1,137 @@
<?php
declare(strict_types=1);
namespace Drupal\materio_ai\Storage;
use Drupal\Component\Datetime\TimeInterface;
use Drupal\Core\Database\Connection;
/**
* Reads and writes the vectors of the materio_ai_embedding table.
*
* Vectors are stored as little-endian 32-bit floats: 4 KB for 1 024 values.
*/
final class EmbeddingStorage {
private const TABLE = 'materio_ai_embedding';
public function __construct(
private readonly Connection $database,
private readonly TimeInterface $time,
) {}
/**
* Inserts or replaces the vector of a node translation.
*
* @param float[] $vector
* The vector.
*/
public function save(int $nid, string $langcode, string $model, string $text_hash, array $vector): void {
$this->database->merge(self::TABLE)
->keys(['nid' => $nid, 'langcode' => $langcode])
->fields([
'model' => $model,
'text_hash' => $text_hash,
'vector' => self::pack($vector),
'changed' => $this->time->getRequestTime(),
])
->execute();
}
/**
* Returns the stored row of a node translation, with the vector unpacked.
*
* @return array{nid: int, langcode: string, model: string, text_hash: string, vector: float[], changed: int}|null
* The row, or NULL when the translation has no vector.
*/
public function load(int $nid, string $langcode): ?array {
$row = $this->database->select(self::TABLE, 'e')
->fields('e')
->condition('nid', $nid)
->condition('langcode', $langcode)
->execute()
->fetchAssoc();
return $row ? self::row($row) : NULL;
}
/**
* Returns the text hashes of a node, keyed by langcode.
*
* Lets the queue skip texts that did not change, without loading vectors.
*
* @return array<string, array{model: string, text_hash: string}>
* The hashes and models.
*/
public function loadHashes(int $nid): array {
return $this->database->select(self::TABLE, 'e')
->fields('e', ['langcode', 'model', 'text_hash'])
->condition('nid', $nid)
->execute()
->fetchAllAssoc('langcode', \PDO::FETCH_ASSOC);
}
/**
* Deletes the vectors of a node, or of one of its translations.
*/
public function delete(int $nid, ?string $langcode = NULL): int {
$query = $this->database->delete(self::TABLE)->condition('nid', $nid);
if ($langcode !== NULL) {
$query->condition('langcode', $langcode);
}
return (int) $query->execute();
}
/**
* Counts the stored vectors, optionally for one model.
*/
public function count(?string $model = NULL): int {
$query = $this->database->select(self::TABLE, 'e');
if ($model !== NULL) {
$query->condition('model', $model);
}
return (int) $query->countQuery()->execute()->fetchField();
}
/**
* Hash of a text, as stored in text_hash.
*/
public static function hash(string $text): string {
return hash('sha256', $text);
}
/**
* Packs a vector as little-endian 32-bit floats.
*
* @param float[] $vector
* The vector.
*/
public static function pack(array $vector): string {
return pack('g*', ...array_values($vector));
}
/**
* Unpacks a vector stored by pack().
*
* @return float[]
* The vector, keys starting at 0.
*/
public static function unpack(string $blob): array {
return array_values(unpack('g*', $blob));
}
/**
* Casts a database row and unpacks its vector.
*/
private static function row(array $row): array {
return [
'nid' => (int) $row['nid'],
'langcode' => $row['langcode'],
'model' => $row['model'],
'text_hash' => $row['text_hash'],
'vector' => self::unpack($row['vector']),
'changed' => (int) $row['changed'],
];
}
}