materio_ai: drush materio-ai:vectorize-all, resumable with progress (V6)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
2026-11-16 21:01:34 +01:00
co-authored by Claude Opus 5.5
parent 7d60678232
commit 2a58602867
@@ -0,0 +1,119 @@
<?php
declare(strict_types=1);
namespace Drupal\materio_ai\Drush\Commands;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\Core\State\StateInterface;
use Drupal\materio_ai\Corpus\CorpusBuilder;
use Drupal\materio_ai\Exception\EmbeddingException;
use Drupal\materio_ai\Storage\EmbeddingStorage;
use Drupal\materio_ai\Vectorizer;
use Drush\Attributes as CLI;
use Drush\Commands\AutowireTrait;
use Drush\Commands\DrushCommands;
/**
* Drush commands of materio_ai.
*/
final class MaterioAiCommands extends DrushCommands {
use AutowireTrait;
/**
* State key holding the last node ID processed by vectorize-all.
*/
private const CURSOR = 'materio_ai.vectorize_all.last_nid';
public function __construct(
private readonly EntityTypeManagerInterface $entityTypeManager,
private readonly StateInterface $state,
private readonly Vectorizer $vectorizer,
private readonly EmbeddingStorage $storage,
) {
parent::__construct();
}
/**
* Vectorizes every material and theme, and resumes after an interruption.
*
* Nodes are processed by ascending ID; the last processed ID is kept in the
* state, so a new run starts after it. Unchanged texts do not call the model.
*/
#[CLI\Command(name: 'materio-ai:vectorize-all', aliases: ['mai-va'])]
#[CLI\Option(name: 'restart', description: 'Start again from the first node.')]
#[CLI\Option(name: 'limit', description: 'Stop after this number of nodes (0: no limit).')]
#[CLI\Usage(name: 'drush mai-va', description: 'Vectorize everything, or resume where the last run stopped.')]
#[CLI\Usage(name: 'drush mai-va --restart', description: 'Start again from the first node (unchanged texts are still skipped).')]
public function vectorizeAll(array $options = ['restart' => FALSE, 'limit' => 0]): int {
if ($options['restart']) {
$this->state->delete(self::CURSOR);
}
$cursor = (int) $this->state->get(self::CURSOR, 0);
$nids = $this->entityTypeManager->getStorage('node')->getQuery()
->accessCheck(FALSE)
->condition('type', CorpusBuilder::BUNDLES, 'IN')
->condition('nid', $cursor, '>')
->sort('nid')
->execute();
$nids = array_map('intval', array_values($nids));
if ($limit = (int) $options['limit']) {
$nids = array_slice($nids, 0, $limit);
}
if (!$nids) {
$this->io()->success(sprintf('Nothing left to vectorize (last node: %d). Use --restart to go through every node again.', $cursor));
return self::EXIT_SUCCESS;
}
if (!$this->vectorizer->isEnabled()) {
$this->io()->note('materio_ai is disabled: vectors are computed, but nothing is queued on save until it is enabled.');
}
$this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor));
$totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
$start = microtime(TRUE);
$nodeStorage = $this->entityTypeManager->getStorage('node');
$bar = $this->io()->createProgressBar(count($nids));
$bar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s% / %estimated:-6s% %message%');
$bar->setMessage('');
$bar->start();
foreach ($nids as $nid) {
try {
$result = $this->vectorizer->vectorize($nid);
}
catch (EmbeddingException $e) {
// Leave the bar where it stopped: finish() would show 100 %.
$this->io()->newLine(2);
$this->io()->error(sprintf('Stopped at node %d: %s Run the command again to resume.', $nid, $e->getMessage()));
$this->summary($totals, $start);
return self::EXIT_FAILURE;
}
foreach ($result as $key => $count) {
$totals[$key] += $count;
}
$this->state->set(self::CURSOR, $nid);
// Keep memory flat over thousands of nodes.
$nodeStorage->resetCache([$nid]);
$bar->setMessage(sprintf('node %d, %d embedded', $nid, $totals['embedded']));
$bar->advance();
}
$bar->finish();
$this->io()->newLine(2);
$this->summary($totals, $start);
return self::EXIT_SUCCESS;
}
/**
* Prints what was done and the number of stored vectors.
*/
private function summary(array $totals, float $start): void {
$seconds = microtime(TRUE) - $start;
$this->io()->text(sprintf(
'Embedded %d texts, %d unchanged, %d vectors deleted, in %s (%.2f embedded texts per second). %d vectors stored.',
$totals['embedded'], $totals['unchanged'], $totals['deleted'],
gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(),
));
}
}