- QueryEmbedder: vector of a search phrase cached 30 days (cache bin materio_ai), model given search_timeout seconds (1 by default, new setting, update 11001), left alone 60 s after a failure (ModelPausedException, not logged). - NeighbourSearch honours the enabled setting: the search called the model even when materio_ai was switched off. - materio_ai_cron() loads the model with the normal timeout: after a restart, loading (2.9 s) exceeds the search delay and Ollama drops it, so searches alone would never load it. Ultimate cron job exported. - On any model failure the search answers with the keyword results only. - QueryEmbedderTest: cache, delay, pause, cache during the pause, warm-up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
108 lines
3.6 KiB
PHP
108 lines
3.6 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace Drupal\materio_ai\Search;
|
|
|
|
use Drupal\Core\Config\ConfigFactoryInterface;
|
|
use Drupal\Core\Entity\EntityTypeManagerInterface;
|
|
use Drupal\Core\Language\LanguageManagerInterface;
|
|
use Drupal\search_api\Utility\Utility;
|
|
|
|
/**
|
|
* Finds the nodes closest in meaning to a phrase (kNN in Solr).
|
|
*
|
|
* Runs a normal Search API query on the database index, tagged so that
|
|
* KnnQuerySubscriber turns it, and only it, into a {!knn} query. Search API
|
|
* processors still add their conditions (content_access: node grants and
|
|
* status for the current user; language), which Solr applies as pre-filters
|
|
* of the kNN search (Solr 9.1+): the vectors also cover nodes the user may
|
|
* not see, only these filters exclude them.
|
|
*/
|
|
final class NeighbourSearch {
|
|
|
|
/**
|
|
* Tag of the Search API query turned into a kNN query.
|
|
*/
|
|
public const TAG = 'materio_ai_knn';
|
|
|
|
/**
|
|
* Query option holding the query vector.
|
|
*/
|
|
public const OPTION_VECTOR = 'materio_ai_knn_vector';
|
|
|
|
/**
|
|
* Search API index holding the vectors (I2).
|
|
*/
|
|
public const INDEX = 'database';
|
|
|
|
public function __construct(
|
|
private readonly EntityTypeManagerInterface $entityTypeManager,
|
|
private readonly LanguageManagerInterface $languageManager,
|
|
private readonly ConfigFactoryInterface $configFactory,
|
|
private readonly QueryEmbedder $embedder,
|
|
) {}
|
|
|
|
/**
|
|
* Nodes closest to a phrase, best first.
|
|
*
|
|
* @param string $phrase
|
|
* The phrase typed by the user.
|
|
* @param string|null $langcode
|
|
* Language of the results; the current language by default.
|
|
* @param int $limit
|
|
* Number of neighbours (topK).
|
|
*
|
|
* @return array<int, float>
|
|
* Scores keyed by nid, best first, none when materio_ai is switched off.
|
|
* Solr score of a cosine kNN query:
|
|
* (1 + cosine) / 2.
|
|
*
|
|
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
|
* When the model fails or is too slow (QueryEmbedder).
|
|
* @throws \Drupal\search_api\SearchApiException
|
|
* When Solr fails.
|
|
*/
|
|
public function search(string $phrase, ?string $langcode = NULL, int $limit = 100): array {
|
|
// Switched off: no text is sent to the model, keyword results only.
|
|
if (!$this->embedder->isEnabled()) {
|
|
return [];
|
|
}
|
|
return $this->searchVector($this->embedder->embed($phrase), $langcode, $limit);
|
|
}
|
|
|
|
/**
|
|
* Nodes closest to a vector, best first (see search()).
|
|
*
|
|
* @param float[] $vector
|
|
* Query vector, of the configured dimension.
|
|
*
|
|
* @return array<int, float>
|
|
* Scores keyed by nid, best first.
|
|
*/
|
|
public function searchVector(array $vector, ?string $langcode = NULL, int $limit = 100): array {
|
|
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
|
|
if (count($vector) !== $dimension || $limit <= 0) {
|
|
return [];
|
|
}
|
|
$index = $this->entityTypeManager->getStorage('search_api_index')->load(self::INDEX);
|
|
$query = $index->query();
|
|
$query->setLanguages([$langcode ?? $this->languageManager->getCurrentLanguage()->getId()]);
|
|
$query->range(0, $limit);
|
|
$query->sort('search_api_relevance', 'DESC');
|
|
$query->setOption(self::OPTION_VECTOR, array_values($vector));
|
|
// Only the item IDs are needed.
|
|
$query->setOption('search_api_retrieved_field_values', ['search_api_id' => 'search_api_id']);
|
|
$query->addTag(self::TAG);
|
|
|
|
$scores = [];
|
|
foreach ($query->execute() as $item_id => $item) {
|
|
[, $raw_id] = Utility::splitCombinedId($item_id);
|
|
$nid = (int) explode(':', $raw_id, 2)[0];
|
|
$scores[$nid] ??= (float) $item->getScore();
|
|
}
|
|
return $scores;
|
|
}
|
|
|
|
}
|