materio_ai: mark a node for reindexing when its vector changes (I3)
Vectorizer::vectorize() calls trackItemsUpdated for each translation whose vector was saved or deleted, also when the model fails half-way. With index_directly the cron indexes them at the end of the request. drush mai-va only marks them (batch tracking) and reminds to run drush sapi-i. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -70,7 +70,10 @@ final class MaterioAiCommands extends DrushCommands {
|
|||||||
}
|
}
|
||||||
$this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor));
|
$this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor));
|
||||||
|
|
||||||
$totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
|
$totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0];
|
||||||
|
// Only mark the changed items: indexing them one node at a time at the
|
||||||
|
// end of the run would hold thousands of items in memory.
|
||||||
|
$this->vectorizer->setDirectIndexing(FALSE);
|
||||||
$start = microtime(TRUE);
|
$start = microtime(TRUE);
|
||||||
$nodeStorage = $this->entityTypeManager->getStorage('node');
|
$nodeStorage = $this->entityTypeManager->getStorage('node');
|
||||||
$bar = $this->io()->createProgressBar(count($nids));
|
$bar = $this->io()->createProgressBar(count($nids));
|
||||||
@@ -114,6 +117,9 @@ final class MaterioAiCommands extends DrushCommands {
|
|||||||
$totals['embedded'], $totals['unchanged'], $totals['deleted'],
|
$totals['embedded'], $totals['unchanged'], $totals['deleted'],
|
||||||
gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(),
|
gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(),
|
||||||
));
|
));
|
||||||
|
if ($totals['marked']) {
|
||||||
|
$this->io()->note(sprintf('%d items marked for reindexing, indexed by cron (cron_limit per run). To index them now: drush sapi-i', $totals['marked']));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ use Drupal\materio_ai\Corpus\CorpusBuilder;
|
|||||||
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
|
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
|
||||||
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||||
use Drupal\node\NodeInterface;
|
use Drupal\node\NodeInterface;
|
||||||
|
use Drupal\search_api\Plugin\search_api\datasource\ContentEntity;
|
||||||
use Drupal\taxonomy\TermInterface;
|
use Drupal\taxonomy\TermInterface;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -26,6 +27,14 @@ final class Vectorizer {
|
|||||||
*/
|
*/
|
||||||
public const QUEUE = 'materio_ai_vectorize';
|
public const QUEUE = 'materio_ai_vectorize';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether Search API indexes the marked items at the end of the request.
|
||||||
|
*
|
||||||
|
* FALSE during the full vectorization: items are only marked, a full
|
||||||
|
* reindex follows (drush sapi-i).
|
||||||
|
*/
|
||||||
|
private bool $directIndexing = TRUE;
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private readonly EntityTypeManagerInterface $entityTypeManager,
|
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||||
private readonly ConfigFactoryInterface $configFactory,
|
private readonly ConfigFactoryInterface $configFactory,
|
||||||
@@ -128,33 +137,95 @@ final class Vectorizer {
|
|||||||
* Deletes the vectors of translations that left the corpus (node deleted,
|
* Deletes the vectors of translations that left the corpus (node deleted,
|
||||||
* unpublished, translation removed), then vectorizes new or changed texts.
|
* unpublished, translation removed), then vectorizes new or changed texts.
|
||||||
*
|
*
|
||||||
* @return array{embedded: int, unchanged: int, deleted: int}
|
* @return array{embedded: int, unchanged: int, deleted: int, marked: int}
|
||||||
* What was done, per translation.
|
* What was done, per translation; marked: items marked for reindexing.
|
||||||
*
|
*
|
||||||
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||||
* When the model fails: vectors already saved for other translations stay.
|
* When the model fails: vectors already saved for other translations stay.
|
||||||
*/
|
*/
|
||||||
public function vectorize(int $nid): array {
|
public function vectorize(int $nid): array {
|
||||||
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
|
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0];
|
||||||
$node = $this->entityTypeManager->getStorage('node')->load($nid);
|
$node = $this->entityTypeManager->getStorage('node')->load($nid);
|
||||||
$texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : [];
|
$texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : [];
|
||||||
$stored = $this->storage->loadHashes($nid);
|
$stored = $this->storage->loadHashes($nid);
|
||||||
|
// Translations whose vector changed: their Solr document is outdated.
|
||||||
|
$changed = [];
|
||||||
|
|
||||||
foreach (array_diff_key($stored, $texts) as $langcode => $row) {
|
try {
|
||||||
$result['deleted'] += $this->storage->delete($nid, (string) $langcode);
|
foreach (array_diff_key($stored, $texts) as $langcode => $row) {
|
||||||
}
|
if ($this->storage->delete($nid, (string) $langcode)) {
|
||||||
|
$result['deleted']++;
|
||||||
$model = $this->client->getModel();
|
$changed[] = (string) $langcode;
|
||||||
foreach ($texts as $langcode => $text) {
|
}
|
||||||
$hash = EmbeddingStorage::hash($text);
|
}
|
||||||
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
|
|
||||||
$result['unchanged']++;
|
$model = $this->client->getModel();
|
||||||
continue;
|
foreach ($texts as $langcode => $text) {
|
||||||
|
$hash = EmbeddingStorage::hash($text);
|
||||||
|
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
|
||||||
|
$result['unchanged']++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
|
||||||
|
$result['embedded']++;
|
||||||
|
$changed[] = (string) $langcode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
// Also when the model fails half-way: vectors already saved are marked.
|
||||||
|
if ($changed && $node instanceof NodeInterface) {
|
||||||
|
$result['marked'] = $this->markForReindex($node, $changed);
|
||||||
}
|
}
|
||||||
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
|
|
||||||
$result['embedded']++;
|
|
||||||
}
|
}
|
||||||
return $result;
|
return $result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Switches indexing at the end of the request on or off (see property).
|
||||||
|
*/
|
||||||
|
public function setDirectIndexing(bool $direct): void {
|
||||||
|
$this->directIndexing = $direct;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Marks translations of a node for reindexing in the indexes holding it.
|
||||||
|
*
|
||||||
|
* The Solr document gets the new vector at the next indexing (I2): at the
|
||||||
|
* end of the request with index_directly, otherwise at the next cron.
|
||||||
|
* A deleted node or translation is removed from the index by Search API.
|
||||||
|
*
|
||||||
|
* @param string[] $langcodes
|
||||||
|
* Translations whose vector changed.
|
||||||
|
*
|
||||||
|
* @return int
|
||||||
|
* Number of items marked, all indexes together.
|
||||||
|
*/
|
||||||
|
private function markForReindex(NodeInterface $node, array $langcodes): int {
|
||||||
|
$ids = [];
|
||||||
|
foreach (array_unique($langcodes) as $langcode) {
|
||||||
|
if ($node->hasTranslation($langcode)) {
|
||||||
|
$ids[] = $node->id() . ':' . $langcode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!$ids) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
$marked = 0;
|
||||||
|
foreach (ContentEntity::getIndexesForEntity($node) as $index) {
|
||||||
|
if (!$this->directIndexing) {
|
||||||
|
$index->startBatchTracking();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$index->trackItemsUpdated('entity:node', $ids);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
if (!$this->directIndexing) {
|
||||||
|
$index->stopBatchTracking();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$marked += count($ids);
|
||||||
|
}
|
||||||
|
return $marked;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user