materio_ai: mark a node for reindexing when its vector changes (I3)

Vectorizer::vectorize() calls trackItemsUpdated for each translation whose
vector was saved or deleted, also when the model fails half-way. With
index_directly the cron indexes them at the end of the request. drush mai-va
only marks them (batch tracking) and reminds to run drush sapi-i.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
2026-11-17 10:40:39 +01:00
co-authored by Claude Opus 5.5
parent 480691fe3e
commit cc19dedf92
2 changed files with 93 additions and 16 deletions
@@ -70,7 +70,10 @@ final class MaterioAiCommands extends DrushCommands {
} }
$this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor)); $this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor));
$totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0]; $totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0];
// Only mark the changed items: indexing them one node at a time at the
// end of the run would hold thousands of items in memory.
$this->vectorizer->setDirectIndexing(FALSE);
$start = microtime(TRUE); $start = microtime(TRUE);
$nodeStorage = $this->entityTypeManager->getStorage('node'); $nodeStorage = $this->entityTypeManager->getStorage('node');
$bar = $this->io()->createProgressBar(count($nids)); $bar = $this->io()->createProgressBar(count($nids));
@@ -114,6 +117,9 @@ final class MaterioAiCommands extends DrushCommands {
$totals['embedded'], $totals['unchanged'], $totals['deleted'], $totals['embedded'], $totals['unchanged'], $totals['deleted'],
gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(), gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(),
)); ));
if ($totals['marked']) {
$this->io()->note(sprintf('%d items marked for reindexing, indexed by cron (cron_limit per run). To index them now: drush sapi-i', $totals['marked']));
}
} }
} }
@@ -11,6 +11,7 @@ use Drupal\materio_ai\Corpus\CorpusBuilder;
use Drupal\materio_ai\Embedding\EmbeddingClientInterface; use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
use Drupal\materio_ai\Storage\EmbeddingStorage; use Drupal\materio_ai\Storage\EmbeddingStorage;
use Drupal\node\NodeInterface; use Drupal\node\NodeInterface;
use Drupal\search_api\Plugin\search_api\datasource\ContentEntity;
use Drupal\taxonomy\TermInterface; use Drupal\taxonomy\TermInterface;
/** /**
@@ -26,6 +27,14 @@ final class Vectorizer {
*/ */
public const QUEUE = 'materio_ai_vectorize'; public const QUEUE = 'materio_ai_vectorize';
/**
* Whether Search API indexes the marked items at the end of the request.
*
* FALSE during the full vectorization: items are only marked, a full
* reindex follows (drush sapi-i).
*/
private bool $directIndexing = TRUE;
public function __construct( public function __construct(
private readonly EntityTypeManagerInterface $entityTypeManager, private readonly EntityTypeManagerInterface $entityTypeManager,
private readonly ConfigFactoryInterface $configFactory, private readonly ConfigFactoryInterface $configFactory,
@@ -128,33 +137,95 @@ final class Vectorizer {
* Deletes the vectors of translations that left the corpus (node deleted, * Deletes the vectors of translations that left the corpus (node deleted,
* unpublished, translation removed), then vectorizes new or changed texts. * unpublished, translation removed), then vectorizes new or changed texts.
* *
* @return array{embedded: int, unchanged: int, deleted: int} * @return array{embedded: int, unchanged: int, deleted: int, marked: int}
* What was done, per translation. * What was done, per translation; marked: items marked for reindexing.
* *
* @throws \Drupal\materio_ai\Exception\EmbeddingException * @throws \Drupal\materio_ai\Exception\EmbeddingException
* When the model fails: vectors already saved for other translations stay. * When the model fails: vectors already saved for other translations stay.
*/ */
public function vectorize(int $nid): array { public function vectorize(int $nid): array {
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0]; $result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0];
$node = $this->entityTypeManager->getStorage('node')->load($nid); $node = $this->entityTypeManager->getStorage('node')->load($nid);
$texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : []; $texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : [];
$stored = $this->storage->loadHashes($nid); $stored = $this->storage->loadHashes($nid);
// Translations whose vector changed: their Solr document is outdated.
$changed = [];
foreach (array_diff_key($stored, $texts) as $langcode => $row) { try {
$result['deleted'] += $this->storage->delete($nid, (string) $langcode); foreach (array_diff_key($stored, $texts) as $langcode => $row) {
} if ($this->storage->delete($nid, (string) $langcode)) {
$result['deleted']++;
$model = $this->client->getModel(); $changed[] = (string) $langcode;
foreach ($texts as $langcode => $text) { }
$hash = EmbeddingStorage::hash($text); }
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
$result['unchanged']++; $model = $this->client->getModel();
continue; foreach ($texts as $langcode => $text) {
$hash = EmbeddingStorage::hash($text);
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
$result['unchanged']++;
continue;
}
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
$result['embedded']++;
$changed[] = (string) $langcode;
}
}
finally {
// Also when the model fails half-way: vectors already saved are marked.
if ($changed && $node instanceof NodeInterface) {
$result['marked'] = $this->markForReindex($node, $changed);
} }
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
$result['embedded']++;
} }
return $result; return $result;
} }
/**
* Switches indexing at the end of the request on or off (see property).
*/
public function setDirectIndexing(bool $direct): void {
$this->directIndexing = $direct;
}
/**
* Marks translations of a node for reindexing in the indexes holding it.
*
* The Solr document gets the new vector at the next indexing (I2): at the
* end of the request with index_directly, otherwise at the next cron.
* A deleted node or translation is removed from the index by Search API.
*
* @param string[] $langcodes
* Translations whose vector changed.
*
* @return int
* Number of items marked, all indexes together.
*/
private function markForReindex(NodeInterface $node, array $langcodes): int {
$ids = [];
foreach (array_unique($langcodes) as $langcode) {
if ($node->hasTranslation($langcode)) {
$ids[] = $node->id() . ':' . $langcode;
}
}
if (!$ids) {
return 0;
}
$marked = 0;
foreach (ContentEntity::getIndexesForEntity($node) as $index) {
if (!$this->directIndexing) {
$index->startBatchTracking();
}
try {
$index->trackItemsUpdated('entity:node', $ids);
}
finally {
if (!$this->directIndexing) {
$index->stopBatchTracking();
}
}
$marked += count($ids);
}
return $marked;
}
} }