materio_ai: drush materio-ai:vectorize-all, resumable with progress (V6)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Drush\Commands;
|
||||
|
||||
use Drupal\Core\Entity\EntityTypeManagerInterface;
|
||||
use Drupal\Core\State\StateInterface;
|
||||
use Drupal\materio_ai\Corpus\CorpusBuilder;
|
||||
use Drupal\materio_ai\Exception\EmbeddingException;
|
||||
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||
use Drupal\materio_ai\Vectorizer;
|
||||
use Drush\Attributes as CLI;
|
||||
use Drush\Commands\AutowireTrait;
|
||||
use Drush\Commands\DrushCommands;
|
||||
|
||||
/**
|
||||
* Drush commands of materio_ai.
|
||||
*/
|
||||
final class MaterioAiCommands extends DrushCommands {
|
||||
|
||||
use AutowireTrait;
|
||||
|
||||
/**
|
||||
* State key holding the last node ID processed by vectorize-all.
|
||||
*/
|
||||
private const CURSOR = 'materio_ai.vectorize_all.last_nid';
|
||||
|
||||
public function __construct(
|
||||
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||
private readonly StateInterface $state,
|
||||
private readonly Vectorizer $vectorizer,
|
||||
private readonly EmbeddingStorage $storage,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Vectorizes every material and theme, and resumes after an interruption.
|
||||
*
|
||||
* Nodes are processed by ascending ID; the last processed ID is kept in the
|
||||
* state, so a new run starts after it. Unchanged texts do not call the model.
|
||||
*/
|
||||
#[CLI\Command(name: 'materio-ai:vectorize-all', aliases: ['mai-va'])]
|
||||
#[CLI\Option(name: 'restart', description: 'Start again from the first node.')]
|
||||
#[CLI\Option(name: 'limit', description: 'Stop after this number of nodes (0: no limit).')]
|
||||
#[CLI\Usage(name: 'drush mai-va', description: 'Vectorize everything, or resume where the last run stopped.')]
|
||||
#[CLI\Usage(name: 'drush mai-va --restart', description: 'Start again from the first node (unchanged texts are still skipped).')]
|
||||
public function vectorizeAll(array $options = ['restart' => FALSE, 'limit' => 0]): int {
|
||||
if ($options['restart']) {
|
||||
$this->state->delete(self::CURSOR);
|
||||
}
|
||||
$cursor = (int) $this->state->get(self::CURSOR, 0);
|
||||
$nids = $this->entityTypeManager->getStorage('node')->getQuery()
|
||||
->accessCheck(FALSE)
|
||||
->condition('type', CorpusBuilder::BUNDLES, 'IN')
|
||||
->condition('nid', $cursor, '>')
|
||||
->sort('nid')
|
||||
->execute();
|
||||
$nids = array_map('intval', array_values($nids));
|
||||
if ($limit = (int) $options['limit']) {
|
||||
$nids = array_slice($nids, 0, $limit);
|
||||
}
|
||||
if (!$nids) {
|
||||
$this->io()->success(sprintf('Nothing left to vectorize (last node: %d). Use --restart to go through every node again.', $cursor));
|
||||
return self::EXIT_SUCCESS;
|
||||
}
|
||||
if (!$this->vectorizer->isEnabled()) {
|
||||
$this->io()->note('materio_ai is disabled: vectors are computed, but nothing is queued on save until it is enabled.');
|
||||
}
|
||||
$this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor));
|
||||
|
||||
$totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0];
|
||||
$start = microtime(TRUE);
|
||||
$nodeStorage = $this->entityTypeManager->getStorage('node');
|
||||
$bar = $this->io()->createProgressBar(count($nids));
|
||||
$bar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s% / %estimated:-6s% %message%');
|
||||
$bar->setMessage('');
|
||||
$bar->start();
|
||||
|
||||
foreach ($nids as $nid) {
|
||||
try {
|
||||
$result = $this->vectorizer->vectorize($nid);
|
||||
}
|
||||
catch (EmbeddingException $e) {
|
||||
// Leave the bar where it stopped: finish() would show 100 %.
|
||||
$this->io()->newLine(2);
|
||||
$this->io()->error(sprintf('Stopped at node %d: %s Run the command again to resume.', $nid, $e->getMessage()));
|
||||
$this->summary($totals, $start);
|
||||
return self::EXIT_FAILURE;
|
||||
}
|
||||
foreach ($result as $key => $count) {
|
||||
$totals[$key] += $count;
|
||||
}
|
||||
$this->state->set(self::CURSOR, $nid);
|
||||
// Keep memory flat over thousands of nodes.
|
||||
$nodeStorage->resetCache([$nid]);
|
||||
$bar->setMessage(sprintf('node %d, %d embedded', $nid, $totals['embedded']));
|
||||
$bar->advance();
|
||||
}
|
||||
$bar->finish();
|
||||
$this->io()->newLine(2);
|
||||
$this->summary($totals, $start);
|
||||
return self::EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints what was done and the number of stored vectors.
|
||||
*/
|
||||
private function summary(array $totals, float $start): void {
|
||||
$seconds = microtime(TRUE) - $start;
|
||||
$this->io()->text(sprintf(
|
||||
'Embedded %d texts, %d unchanged, %d vectors deleted, in %s (%.2f embedded texts per second). %d vectors stored.',
|
||||
$totals['embedded'], $totals['unchanged'], $totals['deleted'],
|
||||
gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(),
|
||||
));
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user