materio_ai: vectorization queue on node save, hash check and retry (V5)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* @file
|
||||
* Hooks of materio_ai: keep the vectors in step with the nodes.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
use Drupal\node\NodeInterface;
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_insert() for node.
|
||||
*/
|
||||
function materio_ai_node_insert(NodeInterface $node): void {
|
||||
\Drupal::service('materio_ai.vectorizer')->enqueue($node);
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_update() for node.
|
||||
*
|
||||
* Also covers a new translation, an unpublished node and a workflow change.
|
||||
*/
|
||||
function materio_ai_node_update(NodeInterface $node): void {
|
||||
\Drupal::service('materio_ai.vectorizer')->enqueue($node);
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_translation_delete() for node.
|
||||
*/
|
||||
function materio_ai_node_translation_delete(NodeInterface $translation): void {
|
||||
\Drupal::service('materio_ai.embedding_storage')->delete((int) $translation->id(), $translation->language()->getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_delete() for node.
|
||||
*
|
||||
* Done at once, even when vectorization is switched off: no model needed.
|
||||
*/
|
||||
function materio_ai_node_delete(NodeInterface $node): void {
|
||||
\Drupal::service('materio_ai.embedding_storage')->delete((int) $node->id());
|
||||
}
|
||||
@@ -17,3 +17,8 @@ services:
|
||||
class: Drupal\materio_ai\Corpus\CorpusBuilder
|
||||
arguments: ['@address.country_repository']
|
||||
Drupal\materio_ai\Corpus\CorpusBuilder: '@materio_ai.corpus_builder'
|
||||
|
||||
materio_ai.vectorizer:
|
||||
class: Drupal\materio_ai\Vectorizer
|
||||
arguments: ['@entity_type.manager', '@config.factory', '@queue', '@materio_ai.corpus_builder', '@materio_ai.embedding_client', '@materio_ai.embedding_storage']
|
||||
Drupal\materio_ai\Vectorizer: '@materio_ai.vectorizer'
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Plugin\QueueWorker;
|
||||
|
||||
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
|
||||
use Drupal\Core\Queue\Attribute\QueueWorker;
|
||||
use Drupal\Core\Queue\QueueWorkerBase;
|
||||
use Drupal\Core\Queue\SuspendQueueException;
|
||||
use Drupal\Core\StringTranslation\TranslatableMarkup;
|
||||
use Drupal\materio_ai\Exception\EmbeddingException;
|
||||
use Drupal\materio_ai\Vectorizer;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\DependencyInjection\ContainerInterface;
|
||||
|
||||
/**
|
||||
* Vectorizes the nodes queued on save, during cron.
|
||||
*
|
||||
* When the model is unreachable, the queue is suspended: the item stays in the
|
||||
* queue and is retried at the next cron run.
|
||||
*/
|
||||
#[QueueWorker(
|
||||
id: Vectorizer::QUEUE,
|
||||
title: new TranslatableMarkup('materiO AI vectorization'),
|
||||
cron: ['time' => 60],
|
||||
)]
|
||||
final class VectorizeNode extends QueueWorkerBase implements ContainerFactoryPluginInterface {
|
||||
|
||||
public function __construct(
|
||||
array $configuration,
|
||||
string $plugin_id,
|
||||
mixed $plugin_definition,
|
||||
private readonly Vectorizer $vectorizer,
|
||||
private readonly LoggerInterface $logger,
|
||||
) {
|
||||
parent::__construct($configuration, $plugin_id, $plugin_definition);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition): self {
|
||||
return new self(
|
||||
$configuration,
|
||||
$plugin_id,
|
||||
$plugin_definition,
|
||||
$container->get('materio_ai.vectorizer'),
|
||||
$container->get('logger.channel.materio_ai'),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function processItem($data): void {
|
||||
if (!$this->vectorizer->isEnabled()) {
|
||||
// Switched off after the item was queued: keep it for later.
|
||||
throw new SuspendQueueException('materiO AI is disabled.');
|
||||
}
|
||||
$nid = (int) ($data['nid'] ?? 0);
|
||||
try {
|
||||
$this->vectorizer->vectorize($nid);
|
||||
}
|
||||
catch (EmbeddingException $e) {
|
||||
$this->logger->warning('Vectorization of node @nid postponed: @message', ['@nid' => $nid, '@message' => $e->getMessage()]);
|
||||
throw new SuspendQueueException($e->getMessage(), 0, $e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\Core\Entity\EntityTypeManagerInterface;
|
||||
use Drupal\Core\Queue\QueueFactory;
|
||||
use Drupal\materio_ai\Corpus\CorpusBuilder;
|
||||
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
|
||||
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||
use Drupal\node\NodeInterface;
|
||||
|
||||
/**
|
||||
* Keeps the vectors of a node in step with its texts.
|
||||
*
|
||||
* Used by the vectorization queue (cron) and by the full vectorization
|
||||
* command. The model is only called for texts whose hash or model changed.
|
||||
*/
|
||||
final class Vectorizer {
|
||||
|
||||
/**
|
||||
* Name of the vectorization queue.
|
||||
*/
|
||||
public const QUEUE = 'materio_ai_vectorize';
|
||||
|
||||
public function __construct(
|
||||
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
private readonly QueueFactory $queueFactory,
|
||||
private readonly CorpusBuilder $corpusBuilder,
|
||||
private readonly EmbeddingClientInterface $client,
|
||||
private readonly EmbeddingStorage $storage,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Tells whether vectorization is switched on (materio_ai.settings:enabled).
|
||||
*/
|
||||
public function isEnabled(): bool {
|
||||
return (bool) $this->configFactory->get('materio_ai.settings')->get('enabled');
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues a node for vectorization, when it is a vectorized type.
|
||||
*/
|
||||
public function enqueue(NodeInterface $node): void {
|
||||
if ($this->isEnabled() && in_array($node->bundle(), CorpusBuilder::BUNDLES, TRUE)) {
|
||||
$this->queueFactory->get(self::QUEUE)->createItem(['nid' => (int) $node->id()]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the vectors of a node.
|
||||
*
|
||||
* Deletes the vectors of translations that left the corpus (node deleted,
|
||||
* unpublished, translation removed), then vectorizes new or changed texts.
|
||||
*
|
||||
* @return array{embedded: int, unchanged: int, deleted: int}
|
||||
* What was done, per translation.
|
||||
*
|
||||
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||
* When the model fails: vectors already saved for other translations stay.
|
||||
*/
|
||||
public function vectorize(int $nid): array {
|
||||
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
|
||||
$node = $this->entityTypeManager->getStorage('node')->load($nid);
|
||||
$texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : [];
|
||||
$stored = $this->storage->loadHashes($nid);
|
||||
|
||||
foreach (array_diff_key($stored, $texts) as $langcode => $row) {
|
||||
$result['deleted'] += $this->storage->delete($nid, (string) $langcode);
|
||||
}
|
||||
|
||||
$model = $this->client->getModel();
|
||||
foreach ($texts as $langcode => $text) {
|
||||
$hash = EmbeddingStorage::hash($text);
|
||||
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
|
||||
$result['unchanged']++;
|
||||
continue;
|
||||
}
|
||||
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
|
||||
$result['embedded']++;
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user