materio_ai: write stored vectors into Solr documents at indexing (I2)
SolrDocumentSubscriber listens to PostCreateIndexDocumentsEvent and sets knn1024_vector on each node document from the materio_ai_embedding table (EmbeddingStorage::loadVectors, one query per batch, active model only). No call to the model: a full reindex does not depend on Ollama. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -28,3 +28,9 @@ services:
|
||||
arguments: ['@config.factory']
|
||||
tags:
|
||||
- { name: event_subscriber }
|
||||
|
||||
materio_ai.solr_document_subscriber:
|
||||
class: Drupal\materio_ai\EventSubscriber\SolrDocumentSubscriber
|
||||
arguments: ['@config.factory', '@materio_ai.embedding_storage']
|
||||
tags:
|
||||
- { name: event_subscriber }
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\EventSubscriber;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\materio_ai\Solr\VectorField;
|
||||
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||
use Drupal\search_api\Utility\Utility;
|
||||
use Drupal\search_api_solr\Event\PostCreateIndexDocumentsEvent;
|
||||
use Drupal\search_api_solr\Event\SearchApiSolrEvents;
|
||||
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
|
||||
|
||||
/**
|
||||
* Writes the stored vector of each node translation into its Solr document.
|
||||
*
|
||||
* Reads the materio_ai_embedding table only: no call to the model, so a full
|
||||
* reindex does not depend on the embedding service. Event of search_api_solr
|
||||
* 4.x, which replaces hook_search_api_solr_documents_alter().
|
||||
*
|
||||
* Independent of the "enabled" setting, which drives vectorization: vectors
|
||||
* already computed are always indexed. A translation without a vector, or with
|
||||
* a vector of another model or dimension, is indexed without the field.
|
||||
*/
|
||||
final class SolrDocumentSubscriber implements EventSubscriberInterface {
|
||||
|
||||
public function __construct(
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
private readonly EmbeddingStorage $storage,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public static function getSubscribedEvents(): array {
|
||||
if (!class_exists(SearchApiSolrEvents::class)) {
|
||||
return [];
|
||||
}
|
||||
return [SearchApiSolrEvents::POST_CREATE_INDEX_DOCUMENTS => 'addVectors'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the vectors to the documents of node items.
|
||||
*/
|
||||
public function addVectors(PostCreateIndexDocumentsEvent $event): void {
|
||||
$config = $this->configFactory->get('materio_ai.settings');
|
||||
$dimension = (int) $config->get('dimension');
|
||||
$model = (string) $config->get('model');
|
||||
if ($dimension <= 0 || $model === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
// Item ID "entity:node/123:fr" => nid 123, langcode fr.
|
||||
$targets = [];
|
||||
foreach ($event->getSearchApiItems() as $item_id => $item) {
|
||||
[$datasource_id, $raw_id] = Utility::splitCombinedId($item_id);
|
||||
if ($datasource_id !== 'entity:node') {
|
||||
continue;
|
||||
}
|
||||
[$nid] = explode(':', $raw_id, 2);
|
||||
$targets[$item_id] = [(int) $nid, $item->getLanguage()];
|
||||
}
|
||||
if (!$targets) {
|
||||
return;
|
||||
}
|
||||
|
||||
$vectors = $this->storage->loadVectors(array_column($targets, 0), $model);
|
||||
if (!$vectors) {
|
||||
return;
|
||||
}
|
||||
|
||||
$field = VectorField::fieldName($dimension);
|
||||
$documents = $event->getSolariumDocuments();
|
||||
foreach ($documents as $document) {
|
||||
$item_id = $document->getFields()['ss_search_api_id'] ?? NULL;
|
||||
if (!isset($targets[$item_id])) {
|
||||
continue;
|
||||
}
|
||||
[$nid, $langcode] = $targets[$item_id];
|
||||
$vector = $vectors[$nid][$langcode] ?? NULL;
|
||||
if ($vector !== NULL && count($vector) === $dimension) {
|
||||
$document->setField($field, $vector);
|
||||
}
|
||||
}
|
||||
$event->setSolariumDocuments($documents);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -55,6 +55,33 @@ final class EmbeddingStorage {
|
||||
return $row ? self::row($row) : NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the vectors of several nodes computed by one model.
|
||||
*
|
||||
* One query for a whole indexing batch.
|
||||
*
|
||||
* @param int[] $nids
|
||||
* Node IDs.
|
||||
*
|
||||
* @return array<int, array<string, float[]>>
|
||||
* Vectors keyed by nid, then langcode.
|
||||
*/
|
||||
public function loadVectors(array $nids, string $model): array {
|
||||
if (!$nids) {
|
||||
return [];
|
||||
}
|
||||
$result = $this->database->select(self::TABLE, 'e')
|
||||
->fields('e', ['nid', 'langcode', 'vector'])
|
||||
->condition('nid', array_values(array_unique($nids)), 'IN')
|
||||
->condition('model', $model)
|
||||
->execute();
|
||||
$vectors = [];
|
||||
foreach ($result as $row) {
|
||||
$vectors[(int) $row->nid][$row->langcode] = self::unpack($row->vector);
|
||||
}
|
||||
return $vectors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the text hashes of a node, keyed by langcode.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user