From cc19dedf920d29880a28692fe574447eb4caf56e Mon Sep 17 00:00:00 2001 From: bach Date: Tue, 17 Nov 2026 10:40:39 +0100 Subject: [PATCH] materio_ai: mark a node for reindexing when its vector changes (I3) Vectorizer::vectorize() calls trackItemsUpdated for each translation whose vector was saved or deleted, also when the model fails half-way. With index_directly the cron indexes them at the end of the request. drush mai-va only marks them (batch tracking) and reminds to run drush sapi-i. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM --- .../src/Drush/Commands/MaterioAiCommands.php | 8 +- .../custom/materio_ai/src/Vectorizer.php | 101 +++++++++++++++--- 2 files changed, 93 insertions(+), 16 deletions(-) diff --git a/web/modules/custom/materio_ai/src/Drush/Commands/MaterioAiCommands.php b/web/modules/custom/materio_ai/src/Drush/Commands/MaterioAiCommands.php index c95edbdf..2346898a 100644 --- a/web/modules/custom/materio_ai/src/Drush/Commands/MaterioAiCommands.php +++ b/web/modules/custom/materio_ai/src/Drush/Commands/MaterioAiCommands.php @@ -70,7 +70,10 @@ final class MaterioAiCommands extends DrushCommands { } $this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor)); - $totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0]; + $totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0]; + // Only mark the changed items: indexing them one node at a time at the + // end of the run would hold thousands of items in memory. + $this->vectorizer->setDirectIndexing(FALSE); $start = microtime(TRUE); $nodeStorage = $this->entityTypeManager->getStorage('node'); $bar = $this->io()->createProgressBar(count($nids)); @@ -114,6 +117,9 @@ final class MaterioAiCommands extends DrushCommands { $totals['embedded'], $totals['unchanged'], $totals['deleted'], gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(), )); + if ($totals['marked']) { + $this->io()->note(sprintf('%d items marked for reindexing, indexed by cron (cron_limit per run). To index them now: drush sapi-i', $totals['marked'])); + } } } diff --git a/web/modules/custom/materio_ai/src/Vectorizer.php b/web/modules/custom/materio_ai/src/Vectorizer.php index bae9ce9c..b35dcb4b 100644 --- a/web/modules/custom/materio_ai/src/Vectorizer.php +++ b/web/modules/custom/materio_ai/src/Vectorizer.php @@ -11,6 +11,7 @@ use Drupal\materio_ai\Corpus\CorpusBuilder; use Drupal\materio_ai\Embedding\EmbeddingClientInterface; use Drupal\materio_ai\Storage\EmbeddingStorage; use Drupal\node\NodeInterface; +use Drupal\search_api\Plugin\search_api\datasource\ContentEntity; use Drupal\taxonomy\TermInterface; /** @@ -26,6 +27,14 @@ final class Vectorizer { */ public const QUEUE = 'materio_ai_vectorize'; + /** + * Whether Search API indexes the marked items at the end of the request. + * + * FALSE during the full vectorization: items are only marked, a full + * reindex follows (drush sapi-i). + */ + private bool $directIndexing = TRUE; + public function __construct( private readonly EntityTypeManagerInterface $entityTypeManager, private readonly ConfigFactoryInterface $configFactory, @@ -128,33 +137,95 @@ final class Vectorizer { * Deletes the vectors of translations that left the corpus (node deleted, * unpublished, translation removed), then vectorizes new or changed texts. * - * @return array{embedded: int, unchanged: int, deleted: int} - * What was done, per translation. + * @return array{embedded: int, unchanged: int, deleted: int, marked: int} + * What was done, per translation; marked: items marked for reindexing. * * @throws \Drupal\materio_ai\Exception\EmbeddingException * When the model fails: vectors already saved for other translations stay. */ public function vectorize(int $nid): array { - $result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0]; + $result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0]; $node = $this->entityTypeManager->getStorage('node')->load($nid); $texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : []; $stored = $this->storage->loadHashes($nid); + // Translations whose vector changed: their Solr document is outdated. + $changed = []; - foreach (array_diff_key($stored, $texts) as $langcode => $row) { - $result['deleted'] += $this->storage->delete($nid, (string) $langcode); - } - - $model = $this->client->getModel(); - foreach ($texts as $langcode => $text) { - $hash = EmbeddingStorage::hash($text); - if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) { - $result['unchanged']++; - continue; + try { + foreach (array_diff_key($stored, $texts) as $langcode => $row) { + if ($this->storage->delete($nid, (string) $langcode)) { + $result['deleted']++; + $changed[] = (string) $langcode; + } + } + + $model = $this->client->getModel(); + foreach ($texts as $langcode => $text) { + $hash = EmbeddingStorage::hash($text); + if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) { + $result['unchanged']++; + continue; + } + $this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text)); + $result['embedded']++; + $changed[] = (string) $langcode; + } + } + finally { + // Also when the model fails half-way: vectors already saved are marked. + if ($changed && $node instanceof NodeInterface) { + $result['marked'] = $this->markForReindex($node, $changed); } - $this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text)); - $result['embedded']++; } return $result; } + /** + * Switches indexing at the end of the request on or off (see property). + */ + public function setDirectIndexing(bool $direct): void { + $this->directIndexing = $direct; + } + + /** + * Marks translations of a node for reindexing in the indexes holding it. + * + * The Solr document gets the new vector at the next indexing (I2): at the + * end of the request with index_directly, otherwise at the next cron. + * A deleted node or translation is removed from the index by Search API. + * + * @param string[] $langcodes + * Translations whose vector changed. + * + * @return int + * Number of items marked, all indexes together. + */ + private function markForReindex(NodeInterface $node, array $langcodes): int { + $ids = []; + foreach (array_unique($langcodes) as $langcode) { + if ($node->hasTranslation($langcode)) { + $ids[] = $node->id() . ':' . $langcode; + } + } + if (!$ids) { + return 0; + } + $marked = 0; + foreach (ContentEntity::getIndexesForEntity($node) as $index) { + if (!$this->directIndexing) { + $index->startBatchTracking(); + } + try { + $index->trackItemsUpdated('entity:node', $ids); + } + finally { + if (!$this->directIndexing) { + $index->stopBatchTracking(); + } + } + $marked += count($ids); + } + return $marked; + } + }