materio_ai: embedding table and storage service (V3)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* @file
|
||||
* Install, update and uninstall functions for materio_ai.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* Implements hook_schema().
|
||||
*/
|
||||
function materio_ai_schema(): array {
|
||||
$schema['materio_ai_embedding'] = [
|
||||
'description' => 'Vector of each node translation, computed by the embedding model.',
|
||||
'fields' => [
|
||||
'nid' => [
|
||||
'type' => 'int',
|
||||
'unsigned' => TRUE,
|
||||
'not null' => TRUE,
|
||||
'description' => 'Node ID.',
|
||||
],
|
||||
'langcode' => [
|
||||
'type' => 'varchar_ascii',
|
||||
'length' => 12,
|
||||
'not null' => TRUE,
|
||||
'description' => 'Language of the vectorized translation.',
|
||||
],
|
||||
'model' => [
|
||||
'type' => 'varchar_ascii',
|
||||
'length' => 128,
|
||||
'not null' => TRUE,
|
||||
'description' => 'Model that computed the vector, for example bge-m3:567m.',
|
||||
],
|
||||
'text_hash' => [
|
||||
'type' => 'varchar_ascii',
|
||||
'length' => 64,
|
||||
'not null' => TRUE,
|
||||
'description' => 'SHA-256 of the vectorized text, to skip unchanged texts.',
|
||||
],
|
||||
'vector' => [
|
||||
'type' => 'blob',
|
||||
'size' => 'normal',
|
||||
'not null' => TRUE,
|
||||
'description' => 'Vector as little-endian 32-bit floats.',
|
||||
],
|
||||
'changed' => [
|
||||
'type' => 'int',
|
||||
'not null' => TRUE,
|
||||
'default' => 0,
|
||||
'description' => 'Unix timestamp of the last vectorization.',
|
||||
],
|
||||
],
|
||||
'primary key' => ['nid', 'langcode'],
|
||||
'indexes' => [
|
||||
'model' => ['model'],
|
||||
],
|
||||
];
|
||||
return $schema;
|
||||
}
|
||||
@@ -7,3 +7,8 @@ services:
|
||||
class: Drupal\materio_ai\Embedding\OllamaEmbeddingClient
|
||||
arguments: ['@http_client', '@config.factory']
|
||||
Drupal\materio_ai\Embedding\EmbeddingClientInterface: '@materio_ai.embedding_client'
|
||||
|
||||
materio_ai.embedding_storage:
|
||||
class: Drupal\materio_ai\Storage\EmbeddingStorage
|
||||
arguments: ['@database', '@datetime.time']
|
||||
Drupal\materio_ai\Storage\EmbeddingStorage: '@materio_ai.embedding_storage'
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Storage;
|
||||
|
||||
use Drupal\Component\Datetime\TimeInterface;
|
||||
use Drupal\Core\Database\Connection;
|
||||
|
||||
/**
|
||||
* Reads and writes the vectors of the materio_ai_embedding table.
|
||||
*
|
||||
* Vectors are stored as little-endian 32-bit floats: 4 KB for 1 024 values.
|
||||
*/
|
||||
final class EmbeddingStorage {
|
||||
|
||||
private const TABLE = 'materio_ai_embedding';
|
||||
|
||||
public function __construct(
|
||||
private readonly Connection $database,
|
||||
private readonly TimeInterface $time,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Inserts or replaces the vector of a node translation.
|
||||
*
|
||||
* @param float[] $vector
|
||||
* The vector.
|
||||
*/
|
||||
public function save(int $nid, string $langcode, string $model, string $text_hash, array $vector): void {
|
||||
$this->database->merge(self::TABLE)
|
||||
->keys(['nid' => $nid, 'langcode' => $langcode])
|
||||
->fields([
|
||||
'model' => $model,
|
||||
'text_hash' => $text_hash,
|
||||
'vector' => self::pack($vector),
|
||||
'changed' => $this->time->getRequestTime(),
|
||||
])
|
||||
->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the stored row of a node translation, with the vector unpacked.
|
||||
*
|
||||
* @return array{nid: int, langcode: string, model: string, text_hash: string, vector: float[], changed: int}|null
|
||||
* The row, or NULL when the translation has no vector.
|
||||
*/
|
||||
public function load(int $nid, string $langcode): ?array {
|
||||
$row = $this->database->select(self::TABLE, 'e')
|
||||
->fields('e')
|
||||
->condition('nid', $nid)
|
||||
->condition('langcode', $langcode)
|
||||
->execute()
|
||||
->fetchAssoc();
|
||||
return $row ? self::row($row) : NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the text hashes of a node, keyed by langcode.
|
||||
*
|
||||
* Lets the queue skip texts that did not change, without loading vectors.
|
||||
*
|
||||
* @return array<string, array{model: string, text_hash: string}>
|
||||
* The hashes and models.
|
||||
*/
|
||||
public function loadHashes(int $nid): array {
|
||||
return $this->database->select(self::TABLE, 'e')
|
||||
->fields('e', ['langcode', 'model', 'text_hash'])
|
||||
->condition('nid', $nid)
|
||||
->execute()
|
||||
->fetchAllAssoc('langcode', \PDO::FETCH_ASSOC);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the vectors of a node, or of one of its translations.
|
||||
*/
|
||||
public function delete(int $nid, ?string $langcode = NULL): int {
|
||||
$query = $this->database->delete(self::TABLE)->condition('nid', $nid);
|
||||
if ($langcode !== NULL) {
|
||||
$query->condition('langcode', $langcode);
|
||||
}
|
||||
return (int) $query->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts the stored vectors, optionally for one model.
|
||||
*/
|
||||
public function count(?string $model = NULL): int {
|
||||
$query = $this->database->select(self::TABLE, 'e');
|
||||
if ($model !== NULL) {
|
||||
$query->condition('model', $model);
|
||||
}
|
||||
return (int) $query->countQuery()->execute()->fetchField();
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash of a text, as stored in text_hash.
|
||||
*/
|
||||
public static function hash(string $text): string {
|
||||
return hash('sha256', $text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Packs a vector as little-endian 32-bit floats.
|
||||
*
|
||||
* @param float[] $vector
|
||||
* The vector.
|
||||
*/
|
||||
public static function pack(array $vector): string {
|
||||
return pack('g*', ...array_values($vector));
|
||||
}
|
||||
|
||||
/**
|
||||
* Unpacks a vector stored by pack().
|
||||
*
|
||||
* @return float[]
|
||||
* The vector, keys starting at 0.
|
||||
*/
|
||||
public static function unpack(string $blob): array {
|
||||
return array_values(unpack('g*', $blob));
|
||||
}
|
||||
|
||||
/**
|
||||
* Casts a database row and unpacks its vector.
|
||||
*/
|
||||
private static function row(array $row): array {
|
||||
return [
|
||||
'nid' => (int) $row['nid'],
|
||||
'langcode' => $row['langcode'],
|
||||
'model' => $row['model'],
|
||||
'text_hash' => $row['text_hash'],
|
||||
'vector' => self::unpack($row['vector']),
|
||||
'changed' => (int) $row['changed'],
|
||||
];
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user