materio_ai: mark a node for reindexing when its vector changes (I3)
Vectorizer::vectorize() calls trackItemsUpdated for each translation whose vector was saved or deleted, also when the model fails half-way. With index_directly the cron indexes them at the end of the request. drush mai-va only marks them (batch tracking) and reminds to run drush sapi-i. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -70,7 +70,10 @@ final class MaterioAiCommands extends DrushCommands {
|
||||
}
|
||||
$this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor));
|
||||
|
||||
$totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
|
||||
$totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0];
|
||||
// Only mark the changed items: indexing them one node at a time at the
|
||||
// end of the run would hold thousands of items in memory.
|
||||
$this->vectorizer->setDirectIndexing(FALSE);
|
||||
$start = microtime(TRUE);
|
||||
$nodeStorage = $this->entityTypeManager->getStorage('node');
|
||||
$bar = $this->io()->createProgressBar(count($nids));
|
||||
@@ -114,6 +117,9 @@ final class MaterioAiCommands extends DrushCommands {
|
||||
$totals['embedded'], $totals['unchanged'], $totals['deleted'],
|
||||
gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(),
|
||||
));
|
||||
if ($totals['marked']) {
|
||||
$this->io()->note(sprintf('%d items marked for reindexing, indexed by cron (cron_limit per run). To index them now: drush sapi-i', $totals['marked']));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use Drupal\materio_ai\Corpus\CorpusBuilder;
|
||||
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
|
||||
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||
use Drupal\node\NodeInterface;
|
||||
use Drupal\search_api\Plugin\search_api\datasource\ContentEntity;
|
||||
use Drupal\taxonomy\TermInterface;
|
||||
|
||||
/**
|
||||
@@ -26,6 +27,14 @@ final class Vectorizer {
|
||||
*/
|
||||
public const QUEUE = 'materio_ai_vectorize';
|
||||
|
||||
/**
|
||||
* Whether Search API indexes the marked items at the end of the request.
|
||||
*
|
||||
* FALSE during the full vectorization: items are only marked, a full
|
||||
* reindex follows (drush sapi-i).
|
||||
*/
|
||||
private bool $directIndexing = TRUE;
|
||||
|
||||
public function __construct(
|
||||
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
@@ -128,33 +137,95 @@ final class Vectorizer {
|
||||
* Deletes the vectors of translations that left the corpus (node deleted,
|
||||
* unpublished, translation removed), then vectorizes new or changed texts.
|
||||
*
|
||||
* @return array{embedded: int, unchanged: int, deleted: int}
|
||||
* What was done, per translation.
|
||||
* @return array{embedded: int, unchanged: int, deleted: int, marked: int}
|
||||
* What was done, per translation; marked: items marked for reindexing.
|
||||
*
|
||||
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||
* When the model fails: vectors already saved for other translations stay.
|
||||
*/
|
||||
public function vectorize(int $nid): array {
|
||||
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
|
||||
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0];
|
||||
$node = $this->entityTypeManager->getStorage('node')->load($nid);
|
||||
$texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : [];
|
||||
$stored = $this->storage->loadHashes($nid);
|
||||
// Translations whose vector changed: their Solr document is outdated.
|
||||
$changed = [];
|
||||
|
||||
foreach (array_diff_key($stored, $texts) as $langcode => $row) {
|
||||
$result['deleted'] += $this->storage->delete($nid, (string) $langcode);
|
||||
}
|
||||
|
||||
$model = $this->client->getModel();
|
||||
foreach ($texts as $langcode => $text) {
|
||||
$hash = EmbeddingStorage::hash($text);
|
||||
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
|
||||
$result['unchanged']++;
|
||||
continue;
|
||||
try {
|
||||
foreach (array_diff_key($stored, $texts) as $langcode => $row) {
|
||||
if ($this->storage->delete($nid, (string) $langcode)) {
|
||||
$result['deleted']++;
|
||||
$changed[] = (string) $langcode;
|
||||
}
|
||||
}
|
||||
|
||||
$model = $this->client->getModel();
|
||||
foreach ($texts as $langcode => $text) {
|
||||
$hash = EmbeddingStorage::hash($text);
|
||||
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
|
||||
$result['unchanged']++;
|
||||
continue;
|
||||
}
|
||||
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
|
||||
$result['embedded']++;
|
||||
$changed[] = (string) $langcode;
|
||||
}
|
||||
}
|
||||
finally {
|
||||
// Also when the model fails half-way: vectors already saved are marked.
|
||||
if ($changed && $node instanceof NodeInterface) {
|
||||
$result['marked'] = $this->markForReindex($node, $changed);
|
||||
}
|
||||
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
|
||||
$result['embedded']++;
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Switches indexing at the end of the request on or off (see property).
|
||||
*/
|
||||
public function setDirectIndexing(bool $direct): void {
|
||||
$this->directIndexing = $direct;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks translations of a node for reindexing in the indexes holding it.
|
||||
*
|
||||
* The Solr document gets the new vector at the next indexing (I2): at the
|
||||
* end of the request with index_directly, otherwise at the next cron.
|
||||
* A deleted node or translation is removed from the index by Search API.
|
||||
*
|
||||
* @param string[] $langcodes
|
||||
* Translations whose vector changed.
|
||||
*
|
||||
* @return int
|
||||
* Number of items marked, all indexes together.
|
||||
*/
|
||||
private function markForReindex(NodeInterface $node, array $langcodes): int {
|
||||
$ids = [];
|
||||
foreach (array_unique($langcodes) as $langcode) {
|
||||
if ($node->hasTranslation($langcode)) {
|
||||
$ids[] = $node->id() . ':' . $langcode;
|
||||
}
|
||||
}
|
||||
if (!$ids) {
|
||||
return 0;
|
||||
}
|
||||
$marked = 0;
|
||||
foreach (ContentEntity::getIndexesForEntity($node) as $index) {
|
||||
if (!$this->directIndexing) {
|
||||
$index->startBatchTracking();
|
||||
}
|
||||
try {
|
||||
$index->trackItemsUpdated('entity:node', $ids);
|
||||
}
|
||||
finally {
|
||||
if (!$this->directIndexing) {
|
||||
$index->stopBatchTracking();
|
||||
}
|
||||
}
|
||||
$marked += count($ids);
|
||||
}
|
||||
return $marked;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user