materio_ai: neighbour search service with Solr kNN (I4)
NeighbourSearch::search() embeds a phrase and runs a Search API query on the
database index, tagged materio_ai_knn. KnnQuerySubscriber turns only this
query into {!knn f=knn1024_vector topK=N}: Search API filter queries
(content_access, language, index) stay as kNN pre-filters (Solr 9.1+), and
keyword queries are untouched. Sent in POST by http_method AUTO.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -34,3 +34,14 @@ services:
|
|||||||
arguments: ['@config.factory', '@materio_ai.embedding_storage']
|
arguments: ['@config.factory', '@materio_ai.embedding_storage']
|
||||||
tags:
|
tags:
|
||||||
- { name: event_subscriber }
|
- { name: event_subscriber }
|
||||||
|
|
||||||
|
materio_ai.neighbour_search:
|
||||||
|
class: Drupal\materio_ai\Search\NeighbourSearch
|
||||||
|
arguments: ['@entity_type.manager', '@language_manager', '@config.factory', '@materio_ai.embedding_client']
|
||||||
|
Drupal\materio_ai\Search\NeighbourSearch: '@materio_ai.neighbour_search'
|
||||||
|
|
||||||
|
materio_ai.knn_query_subscriber:
|
||||||
|
class: Drupal\materio_ai\EventSubscriber\KnnQuerySubscriber
|
||||||
|
arguments: ['@config.factory']
|
||||||
|
tags:
|
||||||
|
- { name: event_subscriber }
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\materio_ai\EventSubscriber;
|
||||||
|
|
||||||
|
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||||
|
use Drupal\materio_ai\Search\NeighbourSearch;
|
||||||
|
use Drupal\materio_ai\Solr\VectorField;
|
||||||
|
use Drupal\search_api_solr\Event\PostConvertedQueryEvent;
|
||||||
|
use Drupal\search_api_solr\Event\SearchApiSolrEvents;
|
||||||
|
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turns the tagged Search API query of NeighbourSearch into a {!knn} query.
|
||||||
|
*
|
||||||
|
* Any other query (keywords, autocomplete, views) is left untouched. The
|
||||||
|
* filter queries built by Search API (index, site, language, access) stay:
|
||||||
|
* Solr 9.1+ applies them as pre-filters of the kNN search. The vector makes
|
||||||
|
* the request ≈ 10 KB: with http_method AUTO, search_api_solr sends it in
|
||||||
|
* POST (Solarium postbigrequest), beyond the 8 KB GET limit of Jetty.
|
||||||
|
*/
|
||||||
|
final class KnnQuerySubscriber implements EventSubscriberInterface {
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private readonly ConfigFactoryInterface $configFactory,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritdoc}
|
||||||
|
*/
|
||||||
|
public static function getSubscribedEvents(): array {
|
||||||
|
if (!class_exists(SearchApiSolrEvents::class)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return [SearchApiSolrEvents::POST_CONVERT_QUERY => 'toKnn'];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Replaces the main query by the kNN query.
|
||||||
|
*/
|
||||||
|
public function toKnn(PostConvertedQueryEvent $event): void {
|
||||||
|
$query = $event->getSearchApiQuery();
|
||||||
|
if (!$query->hasTag(NeighbourSearch::TAG)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$vector = $query->getOption(NeighbourSearch::OPTION_VECTOR);
|
||||||
|
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
|
||||||
|
if (!is_array($vector) || count($vector) !== $dimension) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$limit = $query->getOption('limit') ?: 100;
|
||||||
|
|
||||||
|
$solarium_query = $event->getSolariumQuery();
|
||||||
|
$solarium_query->setQuery(sprintf(
|
||||||
|
'{!knn f=%s topK=%d}[%s]',
|
||||||
|
VectorField::fieldName($dimension),
|
||||||
|
$limit,
|
||||||
|
implode(',', array_map(static fn ($v): string => sprintf('%.7g', $v), $vector)),
|
||||||
|
));
|
||||||
|
// No keyword parser on a kNN query.
|
||||||
|
foreach (['defType', 'qf', 'pf', 'mm', 'tie'] as $param) {
|
||||||
|
$solarium_query->removeParam($param);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\materio_ai\Search;
|
||||||
|
|
||||||
|
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||||
|
use Drupal\Core\Entity\EntityTypeManagerInterface;
|
||||||
|
use Drupal\Core\Language\LanguageManagerInterface;
|
||||||
|
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
|
||||||
|
use Drupal\search_api\Utility\Utility;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Finds the nodes closest in meaning to a phrase (kNN in Solr).
|
||||||
|
*
|
||||||
|
* Runs a normal Search API query on the database index, tagged so that
|
||||||
|
* KnnQuerySubscriber turns it, and only it, into a {!knn} query. Search API
|
||||||
|
* processors still add their conditions (content_access: node grants and
|
||||||
|
* status for the current user; language), which Solr applies as pre-filters
|
||||||
|
* of the kNN search (Solr 9.1+): the vectors also cover nodes the user may
|
||||||
|
* not see, only these filters exclude them.
|
||||||
|
*/
|
||||||
|
final class NeighbourSearch {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tag of the Search API query turned into a kNN query.
|
||||||
|
*/
|
||||||
|
public const TAG = 'materio_ai_knn';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query option holding the query vector.
|
||||||
|
*/
|
||||||
|
public const OPTION_VECTOR = 'materio_ai_knn_vector';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Search API index holding the vectors (I2).
|
||||||
|
*/
|
||||||
|
public const INDEX = 'database';
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||||
|
private readonly LanguageManagerInterface $languageManager,
|
||||||
|
private readonly ConfigFactoryInterface $configFactory,
|
||||||
|
private readonly EmbeddingClientInterface $client,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Nodes closest to a phrase, best first.
|
||||||
|
*
|
||||||
|
* @param string $phrase
|
||||||
|
* The phrase typed by the user.
|
||||||
|
* @param string|null $langcode
|
||||||
|
* Language of the results; the current language by default.
|
||||||
|
* @param int $limit
|
||||||
|
* Number of neighbours (topK).
|
||||||
|
*
|
||||||
|
* @return array<int, float>
|
||||||
|
* Scores keyed by nid, best first. Solr score of a cosine kNN query:
|
||||||
|
* (1 + cosine) / 2.
|
||||||
|
*
|
||||||
|
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||||
|
* When the model fails.
|
||||||
|
* @throws \Drupal\search_api\SearchApiException
|
||||||
|
* When Solr fails.
|
||||||
|
*/
|
||||||
|
public function search(string $phrase, ?string $langcode = NULL, int $limit = 100): array {
|
||||||
|
return $this->searchVector($this->client->embed($phrase), $langcode, $limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Nodes closest to a vector, best first (see search()).
|
||||||
|
*
|
||||||
|
* @param float[] $vector
|
||||||
|
* Query vector, of the configured dimension.
|
||||||
|
*
|
||||||
|
* @return array<int, float>
|
||||||
|
* Scores keyed by nid, best first.
|
||||||
|
*/
|
||||||
|
public function searchVector(array $vector, ?string $langcode = NULL, int $limit = 100): array {
|
||||||
|
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
|
||||||
|
if (count($vector) !== $dimension || $limit <= 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
$index = $this->entityTypeManager->getStorage('search_api_index')->load(self::INDEX);
|
||||||
|
$query = $index->query();
|
||||||
|
$query->setLanguages([$langcode ?? $this->languageManager->getCurrentLanguage()->getId()]);
|
||||||
|
$query->range(0, $limit);
|
||||||
|
$query->sort('search_api_relevance', 'DESC');
|
||||||
|
$query->setOption(self::OPTION_VECTOR, array_values($vector));
|
||||||
|
// Only the item IDs are needed.
|
||||||
|
$query->setOption('search_api_retrieved_field_values', ['search_api_id' => 'search_api_id']);
|
||||||
|
$query->addTag(self::TAG);
|
||||||
|
|
||||||
|
$scores = [];
|
||||||
|
foreach ($query->execute() as $item_id => $item) {
|
||||||
|
[, $raw_id] = Utility::splitCombinedId($item_id);
|
||||||
|
$nid = (int) explode(':', $raw_id, 2)[0];
|
||||||
|
$scores[$nid] ??= (float) $item->getScore();
|
||||||
|
}
|
||||||
|
return $scores;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user