materio_ai: vectorization queue on node save, hash check and retry (V5)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
2026-11-16 20:50:05 +01:00
co-authored by Claude Opus 5.5
parent 96a255fcbc
commit 7d60678232
4 changed files with 206 additions and 0 deletions
@@ -0,0 +1,42 @@
<?php
/**
* @file
* Hooks of materio_ai: keep the vectors in step with the nodes.
*/
declare(strict_types=1);
use Drupal\node\NodeInterface;
/**
* Implements hook_ENTITY_TYPE_insert() for node.
*/
function materio_ai_node_insert(NodeInterface $node): void {
\Drupal::service('materio_ai.vectorizer')->enqueue($node);
}
/**
* Implements hook_ENTITY_TYPE_update() for node.
*
* Also covers a new translation, an unpublished node and a workflow change.
*/
function materio_ai_node_update(NodeInterface $node): void {
\Drupal::service('materio_ai.vectorizer')->enqueue($node);
}
/**
* Implements hook_ENTITY_TYPE_translation_delete() for node.
*/
function materio_ai_node_translation_delete(NodeInterface $translation): void {
\Drupal::service('materio_ai.embedding_storage')->delete((int) $translation->id(), $translation->language()->getId());
}
/**
* Implements hook_ENTITY_TYPE_delete() for node.
*
* Done at once, even when vectorization is switched off: no model needed.
*/
function materio_ai_node_delete(NodeInterface $node): void {
\Drupal::service('materio_ai.embedding_storage')->delete((int) $node->id());
}
@@ -17,3 +17,8 @@ services:
class: Drupal\materio_ai\Corpus\CorpusBuilder
arguments: ['@address.country_repository']
Drupal\materio_ai\Corpus\CorpusBuilder: '@materio_ai.corpus_builder'
materio_ai.vectorizer:
class: Drupal\materio_ai\Vectorizer
arguments: ['@entity_type.manager', '@config.factory', '@queue', '@materio_ai.corpus_builder', '@materio_ai.embedding_client', '@materio_ai.embedding_storage']
Drupal\materio_ai\Vectorizer: '@materio_ai.vectorizer'
@@ -0,0 +1,71 @@
<?php
declare(strict_types=1);
namespace Drupal\materio_ai\Plugin\QueueWorker;
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\Core\Queue\Attribute\QueueWorker;
use Drupal\Core\Queue\QueueWorkerBase;
use Drupal\Core\Queue\SuspendQueueException;
use Drupal\Core\StringTranslation\TranslatableMarkup;
use Drupal\materio_ai\Exception\EmbeddingException;
use Drupal\materio_ai\Vectorizer;
use Psr\Log\LoggerInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Vectorizes the nodes queued on save, during cron.
*
* When the model is unreachable, the queue is suspended: the item stays in the
* queue and is retried at the next cron run.
*/
#[QueueWorker(
id: Vectorizer::QUEUE,
title: new TranslatableMarkup('materiO AI vectorization'),
cron: ['time' => 60],
)]
final class VectorizeNode extends QueueWorkerBase implements ContainerFactoryPluginInterface {
public function __construct(
array $configuration,
string $plugin_id,
mixed $plugin_definition,
private readonly Vectorizer $vectorizer,
private readonly LoggerInterface $logger,
) {
parent::__construct($configuration, $plugin_id, $plugin_definition);
}
/**
* {@inheritdoc}
*/
public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition): self {
return new self(
$configuration,
$plugin_id,
$plugin_definition,
$container->get('materio_ai.vectorizer'),
$container->get('logger.channel.materio_ai'),
);
}
/**
* {@inheritdoc}
*/
public function processItem($data): void {
if (!$this->vectorizer->isEnabled()) {
// Switched off after the item was queued: keep it for later.
throw new SuspendQueueException('materiO AI is disabled.');
}
$nid = (int) ($data['nid'] ?? 0);
try {
$this->vectorizer->vectorize($nid);
}
catch (EmbeddingException $e) {
$this->logger->warning('Vectorization of node @nid postponed: @message', ['@nid' => $nid, '@message' => $e->getMessage()]);
throw new SuspendQueueException($e->getMessage(), 0, $e);
}
}
}
@@ -0,0 +1,88 @@
<?php
declare(strict_types=1);
namespace Drupal\materio_ai;
use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\Core\Queue\QueueFactory;
use Drupal\materio_ai\Corpus\CorpusBuilder;
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
use Drupal\materio_ai\Storage\EmbeddingStorage;
use Drupal\node\NodeInterface;
/**
* Keeps the vectors of a node in step with its texts.
*
* Used by the vectorization queue (cron) and by the full vectorization
* command. The model is only called for texts whose hash or model changed.
*/
final class Vectorizer {
/**
* Name of the vectorization queue.
*/
public const QUEUE = 'materio_ai_vectorize';
public function __construct(
private readonly EntityTypeManagerInterface $entityTypeManager,
private readonly ConfigFactoryInterface $configFactory,
private readonly QueueFactory $queueFactory,
private readonly CorpusBuilder $corpusBuilder,
private readonly EmbeddingClientInterface $client,
private readonly EmbeddingStorage $storage,
) {}
/**
* Tells whether vectorization is switched on (materio_ai.settings:enabled).
*/
public function isEnabled(): bool {
return (bool) $this->configFactory->get('materio_ai.settings')->get('enabled');
}
/**
* Queues a node for vectorization, when it is a vectorized type.
*/
public function enqueue(NodeInterface $node): void {
if ($this->isEnabled() && in_array($node->bundle(), CorpusBuilder::BUNDLES, TRUE)) {
$this->queueFactory->get(self::QUEUE)->createItem(['nid' => (int) $node->id()]);
}
}
/**
* Updates the vectors of a node.
*
* Deletes the vectors of translations that left the corpus (node deleted,
* unpublished, translation removed), then vectorizes new or changed texts.
*
* @return array{embedded: int, unchanged: int, deleted: int}
* What was done, per translation.
*
* @throws \Drupal\materio_ai\Exception\EmbeddingException
* When the model fails: vectors already saved for other translations stay.
*/
public function vectorize(int $nid): array {
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
$node = $this->entityTypeManager->getStorage('node')->load($nid);
$texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : [];
$stored = $this->storage->loadHashes($nid);
foreach (array_diff_key($stored, $texts) as $langcode => $row) {
$result['deleted'] += $this->storage->delete($nid, (string) $langcode);
}
$model = $this->client->getModel();
foreach ($texts as $langcode => $text) {
$hash = EmbeddingStorage::hash($text);
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
$result['unchanged']++;
continue;
}
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
$result['embedded']++;
}
return $result;
}
}