From 2a586028677cd0952b06761c7a14506ed4ab4056 Mon Sep 17 00:00:00 2001 From: bach Date: Mon, 16 Nov 2026 21:01:34 +0100 Subject: [PATCH] materio_ai: drush materio-ai:vectorize-all, resumable with progress (V6) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM --- .../src/Drush/Commands/MaterioAiCommands.php | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 web/modules/custom/materio_ai/src/Drush/Commands/MaterioAiCommands.php diff --git a/web/modules/custom/materio_ai/src/Drush/Commands/MaterioAiCommands.php b/web/modules/custom/materio_ai/src/Drush/Commands/MaterioAiCommands.php new file mode 100644 index 00000000..c95edbdf --- /dev/null +++ b/web/modules/custom/materio_ai/src/Drush/Commands/MaterioAiCommands.php @@ -0,0 +1,119 @@ + FALSE, 'limit' => 0]): int { + if ($options['restart']) { + $this->state->delete(self::CURSOR); + } + $cursor = (int) $this->state->get(self::CURSOR, 0); + $nids = $this->entityTypeManager->getStorage('node')->getQuery() + ->accessCheck(FALSE) + ->condition('type', CorpusBuilder::BUNDLES, 'IN') + ->condition('nid', $cursor, '>') + ->sort('nid') + ->execute(); + $nids = array_map('intval', array_values($nids)); + if ($limit = (int) $options['limit']) { + $nids = array_slice($nids, 0, $limit); + } + if (!$nids) { + $this->io()->success(sprintf('Nothing left to vectorize (last node: %d). Use --restart to go through every node again.', $cursor)); + return self::EXIT_SUCCESS; + } + if (!$this->vectorizer->isEnabled()) { + $this->io()->note('materio_ai is disabled: vectors are computed, but nothing is queued on save until it is enabled.'); + } + $this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor)); + + $totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0]; + $start = microtime(TRUE); + $nodeStorage = $this->entityTypeManager->getStorage('node'); + $bar = $this->io()->createProgressBar(count($nids)); + $bar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s% / %estimated:-6s% %message%'); + $bar->setMessage(''); + $bar->start(); + + foreach ($nids as $nid) { + try { + $result = $this->vectorizer->vectorize($nid); + } + catch (EmbeddingException $e) { + // Leave the bar where it stopped: finish() would show 100 %. + $this->io()->newLine(2); + $this->io()->error(sprintf('Stopped at node %d: %s Run the command again to resume.', $nid, $e->getMessage())); + $this->summary($totals, $start); + return self::EXIT_FAILURE; + } + foreach ($result as $key => $count) { + $totals[$key] += $count; + } + $this->state->set(self::CURSOR, $nid); + // Keep memory flat over thousands of nodes. + $nodeStorage->resetCache([$nid]); + $bar->setMessage(sprintf('node %d, %d embedded', $nid, $totals['embedded'])); + $bar->advance(); + } + $bar->finish(); + $this->io()->newLine(2); + $this->summary($totals, $start); + return self::EXIT_SUCCESS; + } + + /** + * Prints what was done and the number of stored vectors. + */ + private function summary(array $totals, float $start): void { + $seconds = microtime(TRUE) - $start; + $this->io()->text(sprintf( + 'Embedded %d texts, %d unchanged, %d vectors deleted, in %s (%.2f embedded texts per second). %d vectors stored.', + $totals['embedded'], $totals['unchanged'], $totals['deleted'], + gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(), + )); + } + +}