diff --git a/web/modules/custom/materio_ai/materio_ai.services.yml b/web/modules/custom/materio_ai/materio_ai.services.yml index 1e90d0a1..8f538fb9 100644 --- a/web/modules/custom/materio_ai/materio_ai.services.yml +++ b/web/modules/custom/materio_ai/materio_ai.services.yml @@ -34,3 +34,14 @@ services: arguments: ['@config.factory', '@materio_ai.embedding_storage'] tags: - { name: event_subscriber } + + materio_ai.neighbour_search: + class: Drupal\materio_ai\Search\NeighbourSearch + arguments: ['@entity_type.manager', '@language_manager', '@config.factory', '@materio_ai.embedding_client'] + Drupal\materio_ai\Search\NeighbourSearch: '@materio_ai.neighbour_search' + + materio_ai.knn_query_subscriber: + class: Drupal\materio_ai\EventSubscriber\KnnQuerySubscriber + arguments: ['@config.factory'] + tags: + - { name: event_subscriber } diff --git a/web/modules/custom/materio_ai/src/EventSubscriber/KnnQuerySubscriber.php b/web/modules/custom/materio_ai/src/EventSubscriber/KnnQuerySubscriber.php new file mode 100644 index 00000000..0c582a53 --- /dev/null +++ b/web/modules/custom/materio_ai/src/EventSubscriber/KnnQuerySubscriber.php @@ -0,0 +1,67 @@ + 'toKnn']; + } + + /** + * Replaces the main query by the kNN query. + */ + public function toKnn(PostConvertedQueryEvent $event): void { + $query = $event->getSearchApiQuery(); + if (!$query->hasTag(NeighbourSearch::TAG)) { + return; + } + $vector = $query->getOption(NeighbourSearch::OPTION_VECTOR); + $dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension'); + if (!is_array($vector) || count($vector) !== $dimension) { + return; + } + $limit = $query->getOption('limit') ?: 100; + + $solarium_query = $event->getSolariumQuery(); + $solarium_query->setQuery(sprintf( + '{!knn f=%s topK=%d}[%s]', + VectorField::fieldName($dimension), + $limit, + implode(',', array_map(static fn ($v): string => sprintf('%.7g', $v), $vector)), + )); + // No keyword parser on a kNN query. + foreach (['defType', 'qf', 'pf', 'mm', 'tie'] as $param) { + $solarium_query->removeParam($param); + } + } + +} diff --git a/web/modules/custom/materio_ai/src/Search/NeighbourSearch.php b/web/modules/custom/materio_ai/src/Search/NeighbourSearch.php new file mode 100644 index 00000000..ad2fe4f7 --- /dev/null +++ b/web/modules/custom/materio_ai/src/Search/NeighbourSearch.php @@ -0,0 +1,103 @@ + + * Scores keyed by nid, best first. Solr score of a cosine kNN query: + * (1 + cosine) / 2. + * + * @throws \Drupal\materio_ai\Exception\EmbeddingException + * When the model fails. + * @throws \Drupal\search_api\SearchApiException + * When Solr fails. + */ + public function search(string $phrase, ?string $langcode = NULL, int $limit = 100): array { + return $this->searchVector($this->client->embed($phrase), $langcode, $limit); + } + + /** + * Nodes closest to a vector, best first (see search()). + * + * @param float[] $vector + * Query vector, of the configured dimension. + * + * @return array + * Scores keyed by nid, best first. + */ + public function searchVector(array $vector, ?string $langcode = NULL, int $limit = 100): array { + $dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension'); + if (count($vector) !== $dimension || $limit <= 0) { + return []; + } + $index = $this->entityTypeManager->getStorage('search_api_index')->load(self::INDEX); + $query = $index->query(); + $query->setLanguages([$langcode ?? $this->languageManager->getCurrentLanguage()->getId()]); + $query->range(0, $limit); + $query->sort('search_api_relevance', 'DESC'); + $query->setOption(self::OPTION_VECTOR, array_values($vector)); + // Only the item IDs are needed. + $query->setOption('search_api_retrieved_field_values', ['search_api_id' => 'search_api_id']); + $query->addTag(self::TAG); + + $scores = []; + foreach ($query->execute() as $item_id => $item) { + [, $raw_id] = Utility::splitCombinedId($item_id); + $nid = (int) explode(':', $raw_id, 2)[0]; + $scores[$nid] ??= (float) $item->getScore(); + } + return $scores; + } + +}