From 7d6067823280a5f9d090022b714bf312c50a9fd5 Mon Sep 17 00:00:00 2001 From: bach Date: Mon, 16 Nov 2026 20:50:05 +0100 Subject: [PATCH] materio_ai: vectorization queue on node save, hash check and retry (V5) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM --- .../custom/materio_ai/materio_ai.module | 42 +++++++++ .../custom/materio_ai/materio_ai.services.yml | 5 ++ .../src/Plugin/QueueWorker/VectorizeNode.php | 71 +++++++++++++++ .../custom/materio_ai/src/Vectorizer.php | 88 +++++++++++++++++++ 4 files changed, 206 insertions(+) create mode 100644 web/modules/custom/materio_ai/materio_ai.module create mode 100644 web/modules/custom/materio_ai/src/Plugin/QueueWorker/VectorizeNode.php create mode 100644 web/modules/custom/materio_ai/src/Vectorizer.php diff --git a/web/modules/custom/materio_ai/materio_ai.module b/web/modules/custom/materio_ai/materio_ai.module new file mode 100644 index 00000000..b1b19b60 --- /dev/null +++ b/web/modules/custom/materio_ai/materio_ai.module @@ -0,0 +1,42 @@ +enqueue($node); +} + +/** + * Implements hook_ENTITY_TYPE_update() for node. + * + * Also covers a new translation, an unpublished node and a workflow change. + */ +function materio_ai_node_update(NodeInterface $node): void { + \Drupal::service('materio_ai.vectorizer')->enqueue($node); +} + +/** + * Implements hook_ENTITY_TYPE_translation_delete() for node. + */ +function materio_ai_node_translation_delete(NodeInterface $translation): void { + \Drupal::service('materio_ai.embedding_storage')->delete((int) $translation->id(), $translation->language()->getId()); +} + +/** + * Implements hook_ENTITY_TYPE_delete() for node. + * + * Done at once, even when vectorization is switched off: no model needed. + */ +function materio_ai_node_delete(NodeInterface $node): void { + \Drupal::service('materio_ai.embedding_storage')->delete((int) $node->id()); +} diff --git a/web/modules/custom/materio_ai/materio_ai.services.yml b/web/modules/custom/materio_ai/materio_ai.services.yml index 972df21c..ef364471 100644 --- a/web/modules/custom/materio_ai/materio_ai.services.yml +++ b/web/modules/custom/materio_ai/materio_ai.services.yml @@ -17,3 +17,8 @@ services: class: Drupal\materio_ai\Corpus\CorpusBuilder arguments: ['@address.country_repository'] Drupal\materio_ai\Corpus\CorpusBuilder: '@materio_ai.corpus_builder' + + materio_ai.vectorizer: + class: Drupal\materio_ai\Vectorizer + arguments: ['@entity_type.manager', '@config.factory', '@queue', '@materio_ai.corpus_builder', '@materio_ai.embedding_client', '@materio_ai.embedding_storage'] + Drupal\materio_ai\Vectorizer: '@materio_ai.vectorizer' diff --git a/web/modules/custom/materio_ai/src/Plugin/QueueWorker/VectorizeNode.php b/web/modules/custom/materio_ai/src/Plugin/QueueWorker/VectorizeNode.php new file mode 100644 index 00000000..a90a4d49 --- /dev/null +++ b/web/modules/custom/materio_ai/src/Plugin/QueueWorker/VectorizeNode.php @@ -0,0 +1,71 @@ + 60], +)] +final class VectorizeNode extends QueueWorkerBase implements ContainerFactoryPluginInterface { + + public function __construct( + array $configuration, + string $plugin_id, + mixed $plugin_definition, + private readonly Vectorizer $vectorizer, + private readonly LoggerInterface $logger, + ) { + parent::__construct($configuration, $plugin_id, $plugin_definition); + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition): self { + return new self( + $configuration, + $plugin_id, + $plugin_definition, + $container->get('materio_ai.vectorizer'), + $container->get('logger.channel.materio_ai'), + ); + } + + /** + * {@inheritdoc} + */ + public function processItem($data): void { + if (!$this->vectorizer->isEnabled()) { + // Switched off after the item was queued: keep it for later. + throw new SuspendQueueException('materiO AI is disabled.'); + } + $nid = (int) ($data['nid'] ?? 0); + try { + $this->vectorizer->vectorize($nid); + } + catch (EmbeddingException $e) { + $this->logger->warning('Vectorization of node @nid postponed: @message', ['@nid' => $nid, '@message' => $e->getMessage()]); + throw new SuspendQueueException($e->getMessage(), 0, $e); + } + } + +} diff --git a/web/modules/custom/materio_ai/src/Vectorizer.php b/web/modules/custom/materio_ai/src/Vectorizer.php new file mode 100644 index 00000000..1ee00308 --- /dev/null +++ b/web/modules/custom/materio_ai/src/Vectorizer.php @@ -0,0 +1,88 @@ +configFactory->get('materio_ai.settings')->get('enabled'); + } + + /** + * Queues a node for vectorization, when it is a vectorized type. + */ + public function enqueue(NodeInterface $node): void { + if ($this->isEnabled() && in_array($node->bundle(), CorpusBuilder::BUNDLES, TRUE)) { + $this->queueFactory->get(self::QUEUE)->createItem(['nid' => (int) $node->id()]); + } + } + + /** + * Updates the vectors of a node. + * + * Deletes the vectors of translations that left the corpus (node deleted, + * unpublished, translation removed), then vectorizes new or changed texts. + * + * @return array{embedded: int, unchanged: int, deleted: int} + * What was done, per translation. + * + * @throws \Drupal\materio_ai\Exception\EmbeddingException + * When the model fails: vectors already saved for other translations stay. + */ + public function vectorize(int $nid): array { + $result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0]; + $node = $this->entityTypeManager->getStorage('node')->load($nid); + $texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : []; + $stored = $this->storage->loadHashes($nid); + + foreach (array_diff_key($stored, $texts) as $langcode => $row) { + $result['deleted'] += $this->storage->delete($nid, (string) $langcode); + } + + $model = $this->client->getModel(); + foreach ($texts as $langcode => $text) { + $hash = EmbeddingStorage::hash($text); + if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) { + $result['unchanged']++; + continue; + } + $this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text)); + $result['embedded']++; + } + return $result; + } + +}