materio_ai: vectorization queue on node save, hash check and retry (V5)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -0,0 +1,42 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file
|
||||||
|
* Hooks of materio_ai: keep the vectors in step with the nodes.
|
||||||
|
*/
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
use Drupal\node\NodeInterface;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implements hook_ENTITY_TYPE_insert() for node.
|
||||||
|
*/
|
||||||
|
function materio_ai_node_insert(NodeInterface $node): void {
|
||||||
|
\Drupal::service('materio_ai.vectorizer')->enqueue($node);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implements hook_ENTITY_TYPE_update() for node.
|
||||||
|
*
|
||||||
|
* Also covers a new translation, an unpublished node and a workflow change.
|
||||||
|
*/
|
||||||
|
function materio_ai_node_update(NodeInterface $node): void {
|
||||||
|
\Drupal::service('materio_ai.vectorizer')->enqueue($node);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implements hook_ENTITY_TYPE_translation_delete() for node.
|
||||||
|
*/
|
||||||
|
function materio_ai_node_translation_delete(NodeInterface $translation): void {
|
||||||
|
\Drupal::service('materio_ai.embedding_storage')->delete((int) $translation->id(), $translation->language()->getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implements hook_ENTITY_TYPE_delete() for node.
|
||||||
|
*
|
||||||
|
* Done at once, even when vectorization is switched off: no model needed.
|
||||||
|
*/
|
||||||
|
function materio_ai_node_delete(NodeInterface $node): void {
|
||||||
|
\Drupal::service('materio_ai.embedding_storage')->delete((int) $node->id());
|
||||||
|
}
|
||||||
@@ -17,3 +17,8 @@ services:
|
|||||||
class: Drupal\materio_ai\Corpus\CorpusBuilder
|
class: Drupal\materio_ai\Corpus\CorpusBuilder
|
||||||
arguments: ['@address.country_repository']
|
arguments: ['@address.country_repository']
|
||||||
Drupal\materio_ai\Corpus\CorpusBuilder: '@materio_ai.corpus_builder'
|
Drupal\materio_ai\Corpus\CorpusBuilder: '@materio_ai.corpus_builder'
|
||||||
|
|
||||||
|
materio_ai.vectorizer:
|
||||||
|
class: Drupal\materio_ai\Vectorizer
|
||||||
|
arguments: ['@entity_type.manager', '@config.factory', '@queue', '@materio_ai.corpus_builder', '@materio_ai.embedding_client', '@materio_ai.embedding_storage']
|
||||||
|
Drupal\materio_ai\Vectorizer: '@materio_ai.vectorizer'
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\materio_ai\Plugin\QueueWorker;
|
||||||
|
|
||||||
|
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
|
||||||
|
use Drupal\Core\Queue\Attribute\QueueWorker;
|
||||||
|
use Drupal\Core\Queue\QueueWorkerBase;
|
||||||
|
use Drupal\Core\Queue\SuspendQueueException;
|
||||||
|
use Drupal\Core\StringTranslation\TranslatableMarkup;
|
||||||
|
use Drupal\materio_ai\Exception\EmbeddingException;
|
||||||
|
use Drupal\materio_ai\Vectorizer;
|
||||||
|
use Psr\Log\LoggerInterface;
|
||||||
|
use Symfony\Component\DependencyInjection\ContainerInterface;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Vectorizes the nodes queued on save, during cron.
|
||||||
|
*
|
||||||
|
* When the model is unreachable, the queue is suspended: the item stays in the
|
||||||
|
* queue and is retried at the next cron run.
|
||||||
|
*/
|
||||||
|
#[QueueWorker(
|
||||||
|
id: Vectorizer::QUEUE,
|
||||||
|
title: new TranslatableMarkup('materiO AI vectorization'),
|
||||||
|
cron: ['time' => 60],
|
||||||
|
)]
|
||||||
|
final class VectorizeNode extends QueueWorkerBase implements ContainerFactoryPluginInterface {
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
array $configuration,
|
||||||
|
string $plugin_id,
|
||||||
|
mixed $plugin_definition,
|
||||||
|
private readonly Vectorizer $vectorizer,
|
||||||
|
private readonly LoggerInterface $logger,
|
||||||
|
) {
|
||||||
|
parent::__construct($configuration, $plugin_id, $plugin_definition);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritdoc}
|
||||||
|
*/
|
||||||
|
public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition): self {
|
||||||
|
return new self(
|
||||||
|
$configuration,
|
||||||
|
$plugin_id,
|
||||||
|
$plugin_definition,
|
||||||
|
$container->get('materio_ai.vectorizer'),
|
||||||
|
$container->get('logger.channel.materio_ai'),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritdoc}
|
||||||
|
*/
|
||||||
|
public function processItem($data): void {
|
||||||
|
if (!$this->vectorizer->isEnabled()) {
|
||||||
|
// Switched off after the item was queued: keep it for later.
|
||||||
|
throw new SuspendQueueException('materiO AI is disabled.');
|
||||||
|
}
|
||||||
|
$nid = (int) ($data['nid'] ?? 0);
|
||||||
|
try {
|
||||||
|
$this->vectorizer->vectorize($nid);
|
||||||
|
}
|
||||||
|
catch (EmbeddingException $e) {
|
||||||
|
$this->logger->warning('Vectorization of node @nid postponed: @message', ['@nid' => $nid, '@message' => $e->getMessage()]);
|
||||||
|
throw new SuspendQueueException($e->getMessage(), 0, $e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\materio_ai;
|
||||||
|
|
||||||
|
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||||
|
use Drupal\Core\Entity\EntityTypeManagerInterface;
|
||||||
|
use Drupal\Core\Queue\QueueFactory;
|
||||||
|
use Drupal\materio_ai\Corpus\CorpusBuilder;
|
||||||
|
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
|
||||||
|
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||||
|
use Drupal\node\NodeInterface;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keeps the vectors of a node in step with its texts.
|
||||||
|
*
|
||||||
|
* Used by the vectorization queue (cron) and by the full vectorization
|
||||||
|
* command. The model is only called for texts whose hash or model changed.
|
||||||
|
*/
|
||||||
|
final class Vectorizer {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Name of the vectorization queue.
|
||||||
|
*/
|
||||||
|
public const QUEUE = 'materio_ai_vectorize';
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||||
|
private readonly ConfigFactoryInterface $configFactory,
|
||||||
|
private readonly QueueFactory $queueFactory,
|
||||||
|
private readonly CorpusBuilder $corpusBuilder,
|
||||||
|
private readonly EmbeddingClientInterface $client,
|
||||||
|
private readonly EmbeddingStorage $storage,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tells whether vectorization is switched on (materio_ai.settings:enabled).
|
||||||
|
*/
|
||||||
|
public function isEnabled(): bool {
|
||||||
|
return (bool) $this->configFactory->get('materio_ai.settings')->get('enabled');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Queues a node for vectorization, when it is a vectorized type.
|
||||||
|
*/
|
||||||
|
public function enqueue(NodeInterface $node): void {
|
||||||
|
if ($this->isEnabled() && in_array($node->bundle(), CorpusBuilder::BUNDLES, TRUE)) {
|
||||||
|
$this->queueFactory->get(self::QUEUE)->createItem(['nid' => (int) $node->id()]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the vectors of a node.
|
||||||
|
*
|
||||||
|
* Deletes the vectors of translations that left the corpus (node deleted,
|
||||||
|
* unpublished, translation removed), then vectorizes new or changed texts.
|
||||||
|
*
|
||||||
|
* @return array{embedded: int, unchanged: int, deleted: int}
|
||||||
|
* What was done, per translation.
|
||||||
|
*
|
||||||
|
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||||
|
* When the model fails: vectors already saved for other translations stay.
|
||||||
|
*/
|
||||||
|
public function vectorize(int $nid): array {
|
||||||
|
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
|
||||||
|
$node = $this->entityTypeManager->getStorage('node')->load($nid);
|
||||||
|
$texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : [];
|
||||||
|
$stored = $this->storage->loadHashes($nid);
|
||||||
|
|
||||||
|
foreach (array_diff_key($stored, $texts) as $langcode => $row) {
|
||||||
|
$result['deleted'] += $this->storage->delete($nid, (string) $langcode);
|
||||||
|
}
|
||||||
|
|
||||||
|
$model = $this->client->getModel();
|
||||||
|
foreach ($texts as $langcode => $text) {
|
||||||
|
$hash = EmbeddingStorage::hash($text);
|
||||||
|
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
|
||||||
|
$result['unchanged']++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
|
||||||
|
$result['embedded']++;
|
||||||
|
}
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user