materio_ai: neighbour search service with Solr kNN (I4)

NeighbourSearch::search() embeds a phrase and runs a Search API query on the
database index, tagged materio_ai_knn. KnnQuerySubscriber turns only this
query into {!knn f=knn1024_vector topK=N}: Search API filter queries
(content_access, language, index) stay as kNN pre-filters (Solr 9.1+), and
keyword queries are untouched. Sent in POST by http_method AUTO.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
2026-11-17 10:51:32 +01:00
co-authored by Claude Opus 5.5
parent cc19dedf92
commit f1ebea91db
3 changed files with 181 additions and 0 deletions
@@ -34,3 +34,14 @@ services:
arguments: ['@config.factory', '@materio_ai.embedding_storage']
tags:
- { name: event_subscriber }
materio_ai.neighbour_search:
class: Drupal\materio_ai\Search\NeighbourSearch
arguments: ['@entity_type.manager', '@language_manager', '@config.factory', '@materio_ai.embedding_client']
Drupal\materio_ai\Search\NeighbourSearch: '@materio_ai.neighbour_search'
materio_ai.knn_query_subscriber:
class: Drupal\materio_ai\EventSubscriber\KnnQuerySubscriber
arguments: ['@config.factory']
tags:
- { name: event_subscriber }
@@ -0,0 +1,67 @@
<?php
declare(strict_types=1);
namespace Drupal\materio_ai\EventSubscriber;
use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\materio_ai\Search\NeighbourSearch;
use Drupal\materio_ai\Solr\VectorField;
use Drupal\search_api_solr\Event\PostConvertedQueryEvent;
use Drupal\search_api_solr\Event\SearchApiSolrEvents;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
/**
* Turns the tagged Search API query of NeighbourSearch into a {!knn} query.
*
* Any other query (keywords, autocomplete, views) is left untouched. The
* filter queries built by Search API (index, site, language, access) stay:
* Solr 9.1+ applies them as pre-filters of the kNN search. The vector makes
* the request ≈ 10 KB: with http_method AUTO, search_api_solr sends it in
* POST (Solarium postbigrequest), beyond the 8 KB GET limit of Jetty.
*/
final class KnnQuerySubscriber implements EventSubscriberInterface {
public function __construct(
private readonly ConfigFactoryInterface $configFactory,
) {}
/**
* {@inheritdoc}
*/
public static function getSubscribedEvents(): array {
if (!class_exists(SearchApiSolrEvents::class)) {
return [];
}
return [SearchApiSolrEvents::POST_CONVERT_QUERY => 'toKnn'];
}
/**
* Replaces the main query by the kNN query.
*/
public function toKnn(PostConvertedQueryEvent $event): void {
$query = $event->getSearchApiQuery();
if (!$query->hasTag(NeighbourSearch::TAG)) {
return;
}
$vector = $query->getOption(NeighbourSearch::OPTION_VECTOR);
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
if (!is_array($vector) || count($vector) !== $dimension) {
return;
}
$limit = $query->getOption('limit') ?: 100;
$solarium_query = $event->getSolariumQuery();
$solarium_query->setQuery(sprintf(
'{!knn f=%s topK=%d}[%s]',
VectorField::fieldName($dimension),
$limit,
implode(',', array_map(static fn ($v): string => sprintf('%.7g', $v), $vector)),
));
// No keyword parser on a kNN query.
foreach (['defType', 'qf', 'pf', 'mm', 'tie'] as $param) {
$solarium_query->removeParam($param);
}
}
}
@@ -0,0 +1,103 @@
<?php
declare(strict_types=1);
namespace Drupal\materio_ai\Search;
use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\Core\Language\LanguageManagerInterface;
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
use Drupal\search_api\Utility\Utility;
/**
* Finds the nodes closest in meaning to a phrase (kNN in Solr).
*
* Runs a normal Search API query on the database index, tagged so that
* KnnQuerySubscriber turns it, and only it, into a {!knn} query. Search API
* processors still add their conditions (content_access: node grants and
* status for the current user; language), which Solr applies as pre-filters
* of the kNN search (Solr 9.1+): the vectors also cover nodes the user may
* not see, only these filters exclude them.
*/
final class NeighbourSearch {
/**
* Tag of the Search API query turned into a kNN query.
*/
public const TAG = 'materio_ai_knn';
/**
* Query option holding the query vector.
*/
public const OPTION_VECTOR = 'materio_ai_knn_vector';
/**
* Search API index holding the vectors (I2).
*/
public const INDEX = 'database';
public function __construct(
private readonly EntityTypeManagerInterface $entityTypeManager,
private readonly LanguageManagerInterface $languageManager,
private readonly ConfigFactoryInterface $configFactory,
private readonly EmbeddingClientInterface $client,
) {}
/**
* Nodes closest to a phrase, best first.
*
* @param string $phrase
* The phrase typed by the user.
* @param string|null $langcode
* Language of the results; the current language by default.
* @param int $limit
* Number of neighbours (topK).
*
* @return array<int, float>
* Scores keyed by nid, best first. Solr score of a cosine kNN query:
* (1 + cosine) / 2.
*
* @throws \Drupal\materio_ai\Exception\EmbeddingException
* When the model fails.
* @throws \Drupal\search_api\SearchApiException
* When Solr fails.
*/
public function search(string $phrase, ?string $langcode = NULL, int $limit = 100): array {
return $this->searchVector($this->client->embed($phrase), $langcode, $limit);
}
/**
* Nodes closest to a vector, best first (see search()).
*
* @param float[] $vector
* Query vector, of the configured dimension.
*
* @return array<int, float>
* Scores keyed by nid, best first.
*/
public function searchVector(array $vector, ?string $langcode = NULL, int $limit = 100): array {
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
if (count($vector) !== $dimension || $limit <= 0) {
return [];
}
$index = $this->entityTypeManager->getStorage('search_api_index')->load(self::INDEX);
$query = $index->query();
$query->setLanguages([$langcode ?? $this->languageManager->getCurrentLanguage()->getId()]);
$query->range(0, $limit);
$query->sort('search_api_relevance', 'DESC');
$query->setOption(self::OPTION_VECTOR, array_values($vector));
// Only the item IDs are needed.
$query->setOption('search_api_retrieved_field_values', ['search_api_id' => 'search_api_id']);
$query->addTag(self::TAG);
$scores = [];
foreach ($query->execute() as $item_id => $item) {
[, $raw_id] = Utility::splitCombinedId($item_id);
$nid = (int) explode(':', $raw_id, 2)[0];
$scores[$nid] ??= (float) $item->getScore();
}
return $scores;
}
}