Compare commits
17
Commits
master
...
797a5714f1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
797a5714f1 | ||
|
|
8229a80c57 | ||
|
|
58ce1b0513 | ||
|
|
ae69b5c03b | ||
|
|
7db02c8b30 | ||
|
|
f1ebea91db | ||
|
|
cc19dedf92 | ||
|
|
480691fe3e | ||
|
|
ce6124b707 | ||
|
|
b93d791419 | ||
|
|
2a58602867 | ||
|
|
7d60678232 | ||
|
|
96a255fcbc | ||
|
|
3659b9e2a1 | ||
|
|
86ca518c6c | ||
|
|
45f21b2e49 | ||
|
|
890c80eb12 |
@@ -122,6 +122,7 @@ module:
|
||||
maillog: 0
|
||||
masquerade: 0
|
||||
materio_admin: 0
|
||||
materio_ai: 0
|
||||
materio_commerce: 0
|
||||
materio_decoupled: 0
|
||||
materio_expo: 0
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
_core:
|
||||
default_config_hash: __uY4VMv6UzzHCHq1eTjbhXofXp1BTtuR5VjcV_MHYM
|
||||
enabled: false
|
||||
url: 'http://ollama:11434'
|
||||
model: 'bge-m3:567m'
|
||||
dimension: 1024
|
||||
timeout: 10
|
||||
num_thread: 4
|
||||
@@ -0,0 +1,12 @@
|
||||
# Désactivé à l'installation : le module ne fait rien tant que « enabled » est faux.
|
||||
enabled: false
|
||||
# Service Ollama. En local : conteneur « ollama » du docker-compose.
|
||||
# Sur le serveur : à surcharger dans settings.php (CT-materio_embed sur le pont privé).
|
||||
url: 'http://ollama:11434'
|
||||
model: 'bge-m3:567m'
|
||||
dimension: 1024
|
||||
# Délai maximal d'un appel au modèle, en secondes.
|
||||
timeout: 10
|
||||
# Threads de calcul envoyés à chaque appel (options.num_thread) : Ollama ne voit pas
|
||||
# la limite de CPU du conteneur, sans ce réglage il lance trop de threads (mesure M2).
|
||||
num_thread: 4
|
||||
@@ -0,0 +1,22 @@
|
||||
materio_ai.settings:
|
||||
type: config_object
|
||||
label: 'materiO AI settings'
|
||||
mapping:
|
||||
enabled:
|
||||
type: boolean
|
||||
label: 'Enabled'
|
||||
url:
|
||||
type: uri
|
||||
label: 'Embedding service URL'
|
||||
model:
|
||||
type: string
|
||||
label: 'Model'
|
||||
dimension:
|
||||
type: integer
|
||||
label: 'Vector dimension'
|
||||
timeout:
|
||||
type: integer
|
||||
label: 'Timeout (seconds)'
|
||||
num_thread:
|
||||
type: integer
|
||||
label: 'Threads per call'
|
||||
@@ -0,0 +1,12 @@
|
||||
name: 'materiO AI'
|
||||
type: module
|
||||
description: 'Recherche par le sens : vectorisation des fiches avec un modèle d''embedding (bge-m3).'
|
||||
core_version_requirement: ^10.3 || ^11
|
||||
package: 'Materio'
|
||||
configure: materio_ai.settings
|
||||
dependencies:
|
||||
- address:address
|
||||
- drupal:node
|
||||
- drupal:taxonomy
|
||||
- search_api_solr:search_api_solr
|
||||
- workflow:workflow
|
||||
@@ -0,0 +1,60 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* @file
|
||||
* Install, update and uninstall functions for materio_ai.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* Implements hook_schema().
|
||||
*/
|
||||
function materio_ai_schema(): array {
|
||||
$schema['materio_ai_embedding'] = [
|
||||
'description' => 'Vector of each node translation, computed by the embedding model.',
|
||||
'fields' => [
|
||||
'nid' => [
|
||||
'type' => 'int',
|
||||
'unsigned' => TRUE,
|
||||
'not null' => TRUE,
|
||||
'description' => 'Node ID.',
|
||||
],
|
||||
'langcode' => [
|
||||
'type' => 'varchar_ascii',
|
||||
'length' => 12,
|
||||
'not null' => TRUE,
|
||||
'description' => 'Language of the vectorized translation.',
|
||||
],
|
||||
'model' => [
|
||||
'type' => 'varchar_ascii',
|
||||
'length' => 128,
|
||||
'not null' => TRUE,
|
||||
'description' => 'Model that computed the vector, for example bge-m3:567m.',
|
||||
],
|
||||
'text_hash' => [
|
||||
'type' => 'varchar_ascii',
|
||||
'length' => 64,
|
||||
'not null' => TRUE,
|
||||
'description' => 'SHA-256 of the vectorized text, to skip unchanged texts.',
|
||||
],
|
||||
'vector' => [
|
||||
'type' => 'blob',
|
||||
'size' => 'normal',
|
||||
'not null' => TRUE,
|
||||
'description' => 'Vector as little-endian 32-bit floats.',
|
||||
],
|
||||
'changed' => [
|
||||
'type' => 'int',
|
||||
'not null' => TRUE,
|
||||
'default' => 0,
|
||||
'description' => 'Unix timestamp of the last vectorization.',
|
||||
],
|
||||
],
|
||||
'primary key' => ['nid', 'langcode'],
|
||||
'indexes' => [
|
||||
'model' => ['model'],
|
||||
],
|
||||
];
|
||||
return $schema;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
materio_ai.settings:
|
||||
title: 'materiO AI'
|
||||
description: 'Embedding service for the semantic search.'
|
||||
route_name: materio_ai.settings
|
||||
parent: system.admin_config_search
|
||||
@@ -0,0 +1,68 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* @file
|
||||
* Hooks of materio_ai: keep the vectors in step with the nodes.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
use Drupal\node\NodeInterface;
|
||||
use Drupal\taxonomy\TermInterface;
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_insert() for node.
|
||||
*/
|
||||
function materio_ai_node_insert(NodeInterface $node): void {
|
||||
\Drupal::service('materio_ai.vectorizer')->enqueue($node);
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_update() for node.
|
||||
*
|
||||
* Also covers a new translation, an unpublished node and a workflow change.
|
||||
*/
|
||||
function materio_ai_node_update(NodeInterface $node): void {
|
||||
\Drupal::service('materio_ai.vectorizer')->enqueue($node);
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_translation_delete() for node.
|
||||
*/
|
||||
function materio_ai_node_translation_delete(NodeInterface $translation): void {
|
||||
\Drupal::service('materio_ai.embedding_storage')->delete((int) $translation->id(), $translation->language()->getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_delete() for node.
|
||||
*
|
||||
* Done at once, even when vectorization is switched off: no model needed.
|
||||
*/
|
||||
function materio_ai_node_delete(NodeInterface $node): void {
|
||||
\Drupal::service('materio_ai.embedding_storage')->delete((int) $node->id());
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_update() for taxonomy_term.
|
||||
*
|
||||
* A renamed tag, thesaurus term or company changes the text of its nodes.
|
||||
*/
|
||||
function materio_ai_taxonomy_term_update(TermInterface $term): void {
|
||||
\Drupal::service('materio_ai.vectorizer')->enqueueForTerm($term);
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_translation_delete() for taxonomy_term.
|
||||
*/
|
||||
function materio_ai_taxonomy_term_translation_delete(TermInterface $translation): void {
|
||||
\Drupal::service('materio_ai.vectorizer')->enqueueForTerm($translation, TRUE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements hook_ENTITY_TYPE_predelete() for taxonomy_term.
|
||||
*
|
||||
* Predelete: the node references still exist, so the nodes can be found.
|
||||
*/
|
||||
function materio_ai_taxonomy_term_predelete(TermInterface $term): void {
|
||||
\Drupal::service('materio_ai.vectorizer')->enqueueForTerm($term, TRUE);
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
administer materio ai:
|
||||
title: 'Administer materiO AI'
|
||||
description: 'Configure the embedding service used by the semantic search.'
|
||||
restrict access: true
|
||||
@@ -0,0 +1,7 @@
|
||||
materio_ai.settings:
|
||||
path: '/admin/config/search/materio-ai'
|
||||
defaults:
|
||||
_form: '\Drupal\materio_ai\Form\SettingsForm'
|
||||
_title: 'materiO AI'
|
||||
requirements:
|
||||
_permission: 'administer materio ai'
|
||||
@@ -0,0 +1,47 @@
|
||||
services:
|
||||
logger.channel.materio_ai:
|
||||
parent: logger.channel_base
|
||||
arguments: ['materio_ai']
|
||||
|
||||
materio_ai.embedding_client:
|
||||
class: Drupal\materio_ai\Embedding\OllamaEmbeddingClient
|
||||
arguments: ['@http_client', '@config.factory']
|
||||
Drupal\materio_ai\Embedding\EmbeddingClientInterface: '@materio_ai.embedding_client'
|
||||
|
||||
materio_ai.embedding_storage:
|
||||
class: Drupal\materio_ai\Storage\EmbeddingStorage
|
||||
arguments: ['@database', '@datetime.time']
|
||||
Drupal\materio_ai\Storage\EmbeddingStorage: '@materio_ai.embedding_storage'
|
||||
|
||||
materio_ai.corpus_builder:
|
||||
class: Drupal\materio_ai\Corpus\CorpusBuilder
|
||||
arguments: ['@address.country_repository']
|
||||
Drupal\materio_ai\Corpus\CorpusBuilder: '@materio_ai.corpus_builder'
|
||||
|
||||
materio_ai.vectorizer:
|
||||
class: Drupal\materio_ai\Vectorizer
|
||||
arguments: ['@entity_type.manager', '@config.factory', '@queue', '@materio_ai.corpus_builder', '@materio_ai.embedding_client', '@materio_ai.embedding_storage']
|
||||
Drupal\materio_ai\Vectorizer: '@materio_ai.vectorizer'
|
||||
|
||||
materio_ai.solr_config_subscriber:
|
||||
class: Drupal\materio_ai\EventSubscriber\SolrConfigSubscriber
|
||||
arguments: ['@config.factory']
|
||||
tags:
|
||||
- { name: event_subscriber }
|
||||
|
||||
materio_ai.solr_document_subscriber:
|
||||
class: Drupal\materio_ai\EventSubscriber\SolrDocumentSubscriber
|
||||
arguments: ['@config.factory', '@materio_ai.embedding_storage']
|
||||
tags:
|
||||
- { name: event_subscriber }
|
||||
|
||||
materio_ai.neighbour_search:
|
||||
class: Drupal\materio_ai\Search\NeighbourSearch
|
||||
arguments: ['@entity_type.manager', '@language_manager', '@config.factory', '@materio_ai.embedding_client']
|
||||
Drupal\materio_ai\Search\NeighbourSearch: '@materio_ai.neighbour_search'
|
||||
|
||||
materio_ai.knn_query_subscriber:
|
||||
class: Drupal\materio_ai\EventSubscriber\KnnQuerySubscriber
|
||||
arguments: ['@config.factory']
|
||||
tags:
|
||||
- { name: event_subscriber }
|
||||
@@ -0,0 +1,198 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Corpus;
|
||||
|
||||
use CommerceGuys\Addressing\Country\CountryRepositoryInterface;
|
||||
use Drupal\Component\Utility\Html;
|
||||
use Drupal\Core\Entity\ContentEntityInterface;
|
||||
use Drupal\Core\Field\EntityReferenceFieldItemListInterface;
|
||||
use Drupal\node\NodeInterface;
|
||||
|
||||
/**
|
||||
* Builds the text vectorized for each translation of a material or theme.
|
||||
*
|
||||
* Fields follow the Search API "database" index: title, descriptions, tags and
|
||||
* thesaurus with their synonyms, manufacturer and distributor with their
|
||||
* country, plus the family. field_memo (internal notes) is never read.
|
||||
*
|
||||
* Every published node is vectorized, whatever its workflow state: like the
|
||||
* Solr index, access is filtered at query time with the node grants (I4).
|
||||
*/
|
||||
final class CorpusBuilder {
|
||||
|
||||
/**
|
||||
* Node types that are vectorized.
|
||||
*/
|
||||
public const BUNDLES = ['materiau', 'thematique'];
|
||||
|
||||
/**
|
||||
* Languages that are vectorized.
|
||||
*/
|
||||
public const LANGCODES = ['fr', 'en'];
|
||||
|
||||
/**
|
||||
* Material families: field_famille has no translated labels.
|
||||
*/
|
||||
private const FAMILIES = [
|
||||
'fr' => ['W' => 'Bois', 'T' => 'Textile', 'R' => 'Plastique', 'P' => 'Papier', 'C' => 'Composite', 'M' => 'Métal', 'F' => 'Céramique', 'G' => 'Verre', 'S' => 'Béton, pierre', 'O' => 'Autre'],
|
||||
'en' => ['W' => 'Wood', 'T' => 'Textile', 'R' => 'Plastic', 'P' => 'Paper', 'C' => 'Composite', 'M' => 'Metal', 'F' => 'Ceramic', 'G' => 'Glass', 'S' => 'Concrete, stone', 'O' => 'Other'],
|
||||
];
|
||||
|
||||
/**
|
||||
* Labels of the text sections, per language.
|
||||
*/
|
||||
private const LABELS = [
|
||||
'fr' => ['family' => 'Famille', 'tags' => 'Mots-clés', 'thesaurus' => 'Thésaurus', 'manufacturer' => 'Fabricant', 'distributor' => 'Distributeur'],
|
||||
'en' => ['family' => 'Family', 'tags' => 'Keywords', 'thesaurus' => 'Thesaurus', 'manufacturer' => 'Manufacturer', 'distributor' => 'Distributor'],
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
private readonly CountryRepositoryInterface $countryRepository,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Tells whether a node belongs to the corpus.
|
||||
*/
|
||||
public function isEligible(NodeInterface $node): bool {
|
||||
return in_array($node->bundle(), self::BUNDLES, TRUE) && $node->isPublished();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the texts of all vectorized translations of a node.
|
||||
*
|
||||
* @return array<string, string>
|
||||
* Texts keyed by langcode; empty when the node is not eligible.
|
||||
*/
|
||||
public function buildAll(NodeInterface $node): array {
|
||||
if (!$this->isEligible($node)) {
|
||||
return [];
|
||||
}
|
||||
$texts = [];
|
||||
foreach (self::LANGCODES as $langcode) {
|
||||
if ($node->hasTranslation($langcode)) {
|
||||
$translation = $node->getTranslation($langcode);
|
||||
if ($translation->isPublished()) {
|
||||
$texts[$langcode] = $this->build($translation, $langcode);
|
||||
}
|
||||
}
|
||||
}
|
||||
return $texts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the text of one translation.
|
||||
*/
|
||||
public function build(NodeInterface $node, string $langcode): string {
|
||||
$labels = self::LABELS[$langcode] ?? self::LABELS['en'];
|
||||
$parts = [];
|
||||
|
||||
$parts[] = self::clean($node->label() ?? '');
|
||||
if ($node->hasField('field_short_description')) {
|
||||
$parts[] = self::clean((string) $node->get('field_short_description')->value);
|
||||
}
|
||||
if ($node->hasField('body')) {
|
||||
$parts[] = self::clean((string) $node->get('body')->value);
|
||||
}
|
||||
|
||||
if ($node->hasField('field_famille') && ($code = $node->get('field_famille')->value)) {
|
||||
$family = self::FAMILIES[$langcode][$code] ?? self::FAMILIES['en'][$code] ?? NULL;
|
||||
if ($family) {
|
||||
$parts[] = $labels['family'] . ' : ' . $family . '.';
|
||||
}
|
||||
}
|
||||
foreach (['tags' => 'field_tags', 'thesaurus' => 'field_thesaurus'] as $key => $field) {
|
||||
if ($node->hasField($field) && ($names = $this->termNames($node->get($field), $langcode))) {
|
||||
$parts[] = $labels[$key] . ' : ' . implode(', ', $names) . '.';
|
||||
}
|
||||
}
|
||||
foreach (['manufacturer' => 'field_manufacturer', 'distributor' => 'field_distributor'] as $key => $field) {
|
||||
if ($node->hasField($field) && ($companies = $this->companies($node->get($field), $langcode))) {
|
||||
$parts[] = $labels[$key] . ' : ' . implode(', ', $companies) . '.';
|
||||
}
|
||||
}
|
||||
|
||||
return implode("\n", array_filter($parts, fn(string $part): bool => $part !== ''));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the names of referenced terms, with their synonyms, in a language.
|
||||
*
|
||||
* Terms keep the field order: editors put the most important ones first.
|
||||
*
|
||||
* @return string[]
|
||||
* Unique entries such as "name (synonym, synonym)", in field order.
|
||||
*/
|
||||
private function termNames(EntityReferenceFieldItemListInterface $items, string $langcode): array {
|
||||
$names = [];
|
||||
foreach ($items->referencedEntities() as $term) {
|
||||
// Thesaurus names use underscores instead of spaces (isolation_acoustique).
|
||||
$name = self::clean(str_replace('_', ' ', self::translated($term, $langcode)->label() ?? ''));
|
||||
if ($name === '') {
|
||||
continue;
|
||||
}
|
||||
$translated = self::translated($term, $langcode);
|
||||
$synonyms = [];
|
||||
if ($translated->hasField('field_synonyms')) {
|
||||
foreach ($translated->get('field_synonyms') as $item) {
|
||||
$synonym = self::clean(str_replace('_', ' ', (string) $item->value));
|
||||
if ($synonym !== '' && mb_strtolower($synonym) !== mb_strtolower($name)) {
|
||||
$synonyms[$synonym] = $synonym;
|
||||
}
|
||||
}
|
||||
}
|
||||
$names[$name] = $synonyms ? $name . ' (' . implode(', ', $synonyms) . ')' : $name;
|
||||
}
|
||||
return array_values($names);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns "Name (Country)" for each referenced company, or the name alone.
|
||||
*
|
||||
* @return string[]
|
||||
* Unique entries, in field order.
|
||||
*/
|
||||
private function companies(EntityReferenceFieldItemListInterface $items, string $langcode): array {
|
||||
$companies = [];
|
||||
foreach ($items->referencedEntities() as $company) {
|
||||
if (!$company instanceof ContentEntityInterface) {
|
||||
continue;
|
||||
}
|
||||
$name = self::clean(self::translated($company, $langcode)->label() ?? '');
|
||||
if ($name === '') {
|
||||
continue;
|
||||
}
|
||||
$address = $company->hasField('field_public_address') ? $company->get('field_public_address')->first() : NULL;
|
||||
$code = $address?->getCountryCode();
|
||||
if ($code) {
|
||||
try {
|
||||
$name .= ' (' . $this->countryRepository->get($code, $langcode)->getName() . ')';
|
||||
}
|
||||
catch (\Exception) {
|
||||
// Unknown country code: keep the name alone.
|
||||
}
|
||||
}
|
||||
$companies[$name] = $name;
|
||||
}
|
||||
return array_values($companies);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the translation of an entity, or the entity itself.
|
||||
*/
|
||||
private static function translated(ContentEntityInterface $entity, string $langcode): ContentEntityInterface {
|
||||
return $entity->hasTranslation($langcode) ? $entity->getTranslation($langcode) : $entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips HTML and collapses white space.
|
||||
*/
|
||||
private static function clean(string $text): string {
|
||||
$text = Html::decodeEntities(strip_tags(str_replace(['<br', '</p>', '</li>'], ["\n<br", "</p>\n", "</li>\n"], $text)));
|
||||
$text = preg_replace('/[ \t\x{00A0}]+/u', ' ', $text);
|
||||
$text = preg_replace('/\s*\n\s*/u', "\n", $text);
|
||||
return trim((string) $text);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Drush\Commands;
|
||||
|
||||
use Drupal\Core\Entity\EntityTypeManagerInterface;
|
||||
use Drupal\Core\State\StateInterface;
|
||||
use Drupal\materio_ai\Corpus\CorpusBuilder;
|
||||
use Drupal\materio_ai\Exception\EmbeddingException;
|
||||
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||
use Drupal\materio_ai\Vectorizer;
|
||||
use Drush\Attributes as CLI;
|
||||
use Drush\Commands\AutowireTrait;
|
||||
use Drush\Commands\DrushCommands;
|
||||
|
||||
/**
|
||||
* Drush commands of materio_ai.
|
||||
*/
|
||||
final class MaterioAiCommands extends DrushCommands {
|
||||
|
||||
use AutowireTrait;
|
||||
|
||||
/**
|
||||
* State key holding the last node ID processed by vectorize-all.
|
||||
*/
|
||||
private const CURSOR = 'materio_ai.vectorize_all.last_nid';
|
||||
|
||||
public function __construct(
|
||||
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||
private readonly StateInterface $state,
|
||||
private readonly Vectorizer $vectorizer,
|
||||
private readonly EmbeddingStorage $storage,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Vectorizes every material and theme, and resumes after an interruption.
|
||||
*
|
||||
* Nodes are processed by ascending ID; the last processed ID is kept in the
|
||||
* state, so a new run starts after it. Unchanged texts do not call the model.
|
||||
*/
|
||||
#[CLI\Command(name: 'materio-ai:vectorize-all', aliases: ['mai-va'])]
|
||||
#[CLI\Option(name: 'restart', description: 'Start again from the first node.')]
|
||||
#[CLI\Option(name: 'limit', description: 'Stop after this number of nodes (0: no limit).')]
|
||||
#[CLI\Usage(name: 'drush mai-va', description: 'Vectorize everything, or resume where the last run stopped.')]
|
||||
#[CLI\Usage(name: 'drush mai-va --restart', description: 'Start again from the first node (unchanged texts are still skipped).')]
|
||||
public function vectorizeAll(array $options = ['restart' => FALSE, 'limit' => 0]): int {
|
||||
if ($options['restart']) {
|
||||
$this->state->delete(self::CURSOR);
|
||||
}
|
||||
$cursor = (int) $this->state->get(self::CURSOR, 0);
|
||||
$nids = $this->entityTypeManager->getStorage('node')->getQuery()
|
||||
->accessCheck(FALSE)
|
||||
->condition('type', CorpusBuilder::BUNDLES, 'IN')
|
||||
->condition('nid', $cursor, '>')
|
||||
->sort('nid')
|
||||
->execute();
|
||||
$nids = array_map('intval', array_values($nids));
|
||||
if ($limit = (int) $options['limit']) {
|
||||
$nids = array_slice($nids, 0, $limit);
|
||||
}
|
||||
if (!$nids) {
|
||||
$this->io()->success(sprintf('Nothing left to vectorize (last node: %d). Use --restart to go through every node again.', $cursor));
|
||||
return self::EXIT_SUCCESS;
|
||||
}
|
||||
if (!$this->vectorizer->isEnabled()) {
|
||||
$this->io()->note('materio_ai is disabled: vectors are computed, but nothing is queued on save until it is enabled.');
|
||||
}
|
||||
$this->io()->text(sprintf('%d nodes to process, resuming after node %d.', count($nids), $cursor));
|
||||
|
||||
$totals = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0];
|
||||
// Only mark the changed items: indexing them one node at a time at the
|
||||
// end of the run would hold thousands of items in memory.
|
||||
$this->vectorizer->setDirectIndexing(FALSE);
|
||||
$start = microtime(TRUE);
|
||||
$nodeStorage = $this->entityTypeManager->getStorage('node');
|
||||
$bar = $this->io()->createProgressBar(count($nids));
|
||||
$bar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s% / %estimated:-6s% %message%');
|
||||
$bar->setMessage('');
|
||||
$bar->start();
|
||||
|
||||
foreach ($nids as $nid) {
|
||||
try {
|
||||
$result = $this->vectorizer->vectorize($nid);
|
||||
}
|
||||
catch (EmbeddingException $e) {
|
||||
// Leave the bar where it stopped: finish() would show 100 %.
|
||||
$this->io()->newLine(2);
|
||||
$this->io()->error(sprintf('Stopped at node %d: %s Run the command again to resume.', $nid, $e->getMessage()));
|
||||
$this->summary($totals, $start);
|
||||
return self::EXIT_FAILURE;
|
||||
}
|
||||
foreach ($result as $key => $count) {
|
||||
$totals[$key] += $count;
|
||||
}
|
||||
$this->state->set(self::CURSOR, $nid);
|
||||
// Keep memory flat over thousands of nodes.
|
||||
$nodeStorage->resetCache([$nid]);
|
||||
$bar->setMessage(sprintf('node %d, %d embedded', $nid, $totals['embedded']));
|
||||
$bar->advance();
|
||||
}
|
||||
$bar->finish();
|
||||
$this->io()->newLine(2);
|
||||
$this->summary($totals, $start);
|
||||
return self::EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints what was done and the number of stored vectors.
|
||||
*/
|
||||
private function summary(array $totals, float $start): void {
|
||||
$seconds = microtime(TRUE) - $start;
|
||||
$this->io()->text(sprintf(
|
||||
'Embedded %d texts, %d unchanged, %d vectors deleted, in %s (%.2f embedded texts per second). %d vectors stored.',
|
||||
$totals['embedded'], $totals['unchanged'], $totals['deleted'],
|
||||
gmdate('H:i:s', (int) $seconds), $totals['embedded'] / max($seconds, 0.001), $this->storage->count(),
|
||||
));
|
||||
if ($totals['marked']) {
|
||||
$this->io()->note(sprintf('%d items marked for reindexing, indexed by cron (cron_limit per run). To index them now: drush sapi-i', $totals['marked']));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Embedding;
|
||||
|
||||
/**
|
||||
* Turns texts into vectors with the embedding model.
|
||||
*/
|
||||
interface EmbeddingClientInterface {
|
||||
|
||||
/**
|
||||
* Returns the vector of one text.
|
||||
*
|
||||
* @return float[]
|
||||
* A vector of getDimension() values.
|
||||
*
|
||||
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||
*/
|
||||
public function embed(string $text): array;
|
||||
|
||||
/**
|
||||
* Returns the vectors of several texts, in the same order and with the same keys.
|
||||
*
|
||||
* Texts are sent one per call: Ollama gains nothing from batches on CPU
|
||||
* (measured in M2).
|
||||
*
|
||||
* @param string[] $texts
|
||||
* The texts to vectorize.
|
||||
*
|
||||
* @return float[][]
|
||||
* The vectors, keyed like $texts.
|
||||
*
|
||||
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||
*/
|
||||
public function embedMultiple(array $texts): array;
|
||||
|
||||
/**
|
||||
* Returns the model name, stored with each vector.
|
||||
*/
|
||||
public function getModel(): string;
|
||||
|
||||
/**
|
||||
* Returns the expected vector dimension.
|
||||
*/
|
||||
public function getDimension(): int;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Embedding;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\Core\Config\ImmutableConfig;
|
||||
use Drupal\materio_ai\Exception\EmbeddingException;
|
||||
use GuzzleHttp\ClientInterface;
|
||||
use GuzzleHttp\Exception\GuzzleException;
|
||||
|
||||
/**
|
||||
* Embedding client for an Ollama service (POST /api/embed).
|
||||
*/
|
||||
final class OllamaEmbeddingClient implements EmbeddingClientInterface {
|
||||
|
||||
public function __construct(
|
||||
private readonly ClientInterface $httpClient,
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function embed(string $text): array {
|
||||
$text = trim($text);
|
||||
if ($text === '') {
|
||||
throw new EmbeddingException('Cannot vectorize an empty text.');
|
||||
}
|
||||
$config = $this->config();
|
||||
$timeout = max(1, (int) $config->get('timeout'));
|
||||
$url = rtrim((string) $config->get('url'), '/') . '/api/embed';
|
||||
|
||||
try {
|
||||
$response = $this->httpClient->request('POST', $url, [
|
||||
'json' => [
|
||||
'model' => $this->getModel(),
|
||||
'input' => $text,
|
||||
// Ollama counts the cores of the host, not the CPU limit of its
|
||||
// container: without this option it starts too many threads.
|
||||
'options' => ['num_thread' => max(1, (int) $config->get('num_thread'))],
|
||||
],
|
||||
'timeout' => $timeout,
|
||||
'connect_timeout' => min($timeout, 3),
|
||||
]);
|
||||
}
|
||||
catch (GuzzleException $e) {
|
||||
throw new EmbeddingException(sprintf('Embedding service %s failed: %s', $url, $e->getMessage()), 0, $e);
|
||||
}
|
||||
|
||||
try {
|
||||
$data = json_decode((string) $response->getBody(), TRUE, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
catch (\JsonException $e) {
|
||||
throw new EmbeddingException(sprintf('Embedding service %s returned invalid JSON.', $url), 0, $e);
|
||||
}
|
||||
|
||||
$vector = $data['embeddings'][0] ?? NULL;
|
||||
if (!is_array($vector)) {
|
||||
throw new EmbeddingException(sprintf('Embedding service %s returned no vector.', $url));
|
||||
}
|
||||
$dimension = $this->getDimension();
|
||||
if (count($vector) !== $dimension) {
|
||||
throw new EmbeddingException(sprintf('Expected a vector of %d values, got %d (model %s).', $dimension, count($vector), $this->getModel()));
|
||||
}
|
||||
foreach ($vector as $value) {
|
||||
if (!is_int($value) && !is_float($value)) {
|
||||
throw new EmbeddingException('The vector contains a non numeric value.');
|
||||
}
|
||||
}
|
||||
return array_map('floatval', $vector);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function embedMultiple(array $texts): array {
|
||||
$vectors = [];
|
||||
foreach ($texts as $key => $text) {
|
||||
$vectors[$key] = $this->embed($text);
|
||||
}
|
||||
return $vectors;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function getModel(): string {
|
||||
return (string) $this->config()->get('model');
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function getDimension(): int {
|
||||
return (int) $this->config()->get('dimension');
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the module settings.
|
||||
*/
|
||||
private function config(): ImmutableConfig {
|
||||
return $this->configFactory->get('materio_ai.settings');
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\EventSubscriber;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\materio_ai\Search\NeighbourSearch;
|
||||
use Drupal\materio_ai\Solr\VectorField;
|
||||
use Drupal\search_api_solr\Event\PostConvertedQueryEvent;
|
||||
use Drupal\search_api_solr\Event\SearchApiSolrEvents;
|
||||
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
|
||||
|
||||
/**
|
||||
* Turns the tagged Search API query of NeighbourSearch into a {!knn} query.
|
||||
*
|
||||
* Any other query (keywords, autocomplete, views) is left untouched. The
|
||||
* filter queries built by Search API (index, site, language, access) stay:
|
||||
* Solr 9.1+ applies them as pre-filters of the kNN search. The vector makes
|
||||
* the request ≈ 10 KB: with http_method AUTO, search_api_solr sends it in
|
||||
* POST (Solarium postbigrequest), beyond the 8 KB GET limit of Jetty.
|
||||
*/
|
||||
final class KnnQuerySubscriber implements EventSubscriberInterface {
|
||||
|
||||
public function __construct(
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public static function getSubscribedEvents(): array {
|
||||
if (!class_exists(SearchApiSolrEvents::class)) {
|
||||
return [];
|
||||
}
|
||||
return [SearchApiSolrEvents::POST_CONVERT_QUERY => 'toKnn'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the main query by the kNN query.
|
||||
*/
|
||||
public function toKnn(PostConvertedQueryEvent $event): void {
|
||||
$query = $event->getSearchApiQuery();
|
||||
if (!$query->hasTag(NeighbourSearch::TAG)) {
|
||||
return;
|
||||
}
|
||||
$vector = $query->getOption(NeighbourSearch::OPTION_VECTOR);
|
||||
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
|
||||
if (!is_array($vector) || count($vector) !== $dimension) {
|
||||
return;
|
||||
}
|
||||
$limit = $query->getOption('limit') ?: 100;
|
||||
|
||||
$solarium_query = $event->getSolariumQuery();
|
||||
$solarium_query->setQuery(sprintf(
|
||||
'{!knn f=%s topK=%d}[%s]',
|
||||
VectorField::fieldName($dimension),
|
||||
$limit,
|
||||
implode(',', array_map(static fn ($v): string => sprintf('%.7g', $v), $vector)),
|
||||
));
|
||||
// No keyword parser on a kNN query.
|
||||
foreach (['defType', 'qf', 'pf', 'mm', 'tie'] as $param) {
|
||||
$solarium_query->removeParam($param);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\EventSubscriber;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\materio_ai\Solr\VectorField;
|
||||
use Drupal\search_api_solr\Event\PostConfigFilesGenerationEvent;
|
||||
use Drupal\search_api_solr\Event\SearchApiSolrEvents;
|
||||
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
|
||||
|
||||
/**
|
||||
* Adds the dense vector field to the generated Solr configset.
|
||||
*
|
||||
* Event of search_api_solr 4.x, which replaces
|
||||
* hook_search_api_solr_config_files_alter(). Solr 9.1 at least: pre-filters
|
||||
* (fq) apply to the kNN search only from this version.
|
||||
*/
|
||||
final class SolrConfigSubscriber implements EventSubscriberInterface {
|
||||
|
||||
public function __construct(
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public static function getSubscribedEvents(): array {
|
||||
if (!class_exists(SearchApiSolrEvents::class)) {
|
||||
return [];
|
||||
}
|
||||
return [SearchApiSolrEvents::POST_CONFIG_FILES_GENERATION => 'addVectorField'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends the field type and the dynamic field to the schema_extra files.
|
||||
*/
|
||||
public function addVectorField(PostConfigFilesGenerationEvent $event): void {
|
||||
if (version_compare($event->getLuceneMatchVersion(), '9.1', '<')) {
|
||||
return;
|
||||
}
|
||||
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
|
||||
if ($dimension <= 0) {
|
||||
return;
|
||||
}
|
||||
$files = $event->getConfigFiles();
|
||||
foreach ([
|
||||
'schema_extra_types.xml' => VectorField::typeXml($dimension),
|
||||
'schema_extra_fields.xml' => VectorField::dynamicFieldXml($dimension),
|
||||
] as $file => $xml) {
|
||||
if (isset($files[$file]) && !str_contains($files[$file], $xml)) {
|
||||
$files[$file] .= $xml;
|
||||
}
|
||||
}
|
||||
$event->setConfigFiles($files);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\EventSubscriber;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\materio_ai\Solr\VectorField;
|
||||
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||
use Drupal\search_api\Utility\Utility;
|
||||
use Drupal\search_api_solr\Event\PostCreateIndexDocumentsEvent;
|
||||
use Drupal\search_api_solr\Event\SearchApiSolrEvents;
|
||||
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
|
||||
|
||||
/**
|
||||
* Writes the stored vector of each node translation into its Solr document.
|
||||
*
|
||||
* Reads the materio_ai_embedding table only: no call to the model, so a full
|
||||
* reindex does not depend on the embedding service. Event of search_api_solr
|
||||
* 4.x, which replaces hook_search_api_solr_documents_alter().
|
||||
*
|
||||
* Independent of the "enabled" setting, which drives vectorization: vectors
|
||||
* already computed are always indexed. A translation without a vector, or with
|
||||
* a vector of another model or dimension, is indexed without the field.
|
||||
*/
|
||||
final class SolrDocumentSubscriber implements EventSubscriberInterface {
|
||||
|
||||
public function __construct(
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
private readonly EmbeddingStorage $storage,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public static function getSubscribedEvents(): array {
|
||||
if (!class_exists(SearchApiSolrEvents::class)) {
|
||||
return [];
|
||||
}
|
||||
return [SearchApiSolrEvents::POST_CREATE_INDEX_DOCUMENTS => 'addVectors'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the vectors to the documents of node items.
|
||||
*/
|
||||
public function addVectors(PostCreateIndexDocumentsEvent $event): void {
|
||||
$config = $this->configFactory->get('materio_ai.settings');
|
||||
$dimension = (int) $config->get('dimension');
|
||||
$model = (string) $config->get('model');
|
||||
if ($dimension <= 0 || $model === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
// Item ID "entity:node/123:fr" => nid 123, langcode fr.
|
||||
$targets = [];
|
||||
foreach ($event->getSearchApiItems() as $item_id => $item) {
|
||||
[$datasource_id, $raw_id] = Utility::splitCombinedId($item_id);
|
||||
if ($datasource_id !== 'entity:node') {
|
||||
continue;
|
||||
}
|
||||
[$nid] = explode(':', $raw_id, 2);
|
||||
$targets[$item_id] = [(int) $nid, $item->getLanguage()];
|
||||
}
|
||||
if (!$targets) {
|
||||
return;
|
||||
}
|
||||
|
||||
$vectors = $this->storage->loadVectors(array_column($targets, 0), $model);
|
||||
if (!$vectors) {
|
||||
return;
|
||||
}
|
||||
|
||||
$field = VectorField::fieldName($dimension);
|
||||
$documents = $event->getSolariumDocuments();
|
||||
foreach ($documents as $document) {
|
||||
$item_id = $document->getFields()['ss_search_api_id'] ?? NULL;
|
||||
if (!isset($targets[$item_id])) {
|
||||
continue;
|
||||
}
|
||||
[$nid, $langcode] = $targets[$item_id];
|
||||
$vector = $vectors[$nid][$langcode] ?? NULL;
|
||||
if ($vector !== NULL && count($vector) === $dimension) {
|
||||
$document->setField($field, $vector);
|
||||
}
|
||||
}
|
||||
$event->setSolariumDocuments($documents);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Exception;
|
||||
|
||||
/**
|
||||
* The embedding service could not return a valid vector.
|
||||
*
|
||||
* Covers an unreachable or stopped service, a timeout, an HTTP error, an
|
||||
* unexpected response and a vector of the wrong dimension.
|
||||
*/
|
||||
class EmbeddingException extends \RuntimeException {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Form;
|
||||
|
||||
use Drupal\Core\Form\ConfigFormBase;
|
||||
use Drupal\Core\Form\FormStateInterface;
|
||||
|
||||
/**
|
||||
* Settings of the embedding service.
|
||||
*/
|
||||
final class SettingsForm extends ConfigFormBase {
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function getFormId(): string {
|
||||
return 'materio_ai_settings';
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
protected function getEditableConfigNames(): array {
|
||||
return ['materio_ai.settings'];
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function buildForm(array $form, FormStateInterface $form_state): array {
|
||||
$form['enabled'] = [
|
||||
'#type' => 'checkbox',
|
||||
'#title' => $this->t('Enabled'),
|
||||
'#description' => $this->t('When disabled, no text is sent to the model and nothing is vectorized.'),
|
||||
'#config_target' => 'materio_ai.settings:enabled',
|
||||
];
|
||||
$form['url'] = [
|
||||
'#type' => 'url',
|
||||
'#title' => $this->t('Embedding service URL'),
|
||||
'#description' => $this->t('Ollama base URL, for example http://ollama:11434. Usually overridden in settings.php per environment.'),
|
||||
'#required' => TRUE,
|
||||
'#config_target' => 'materio_ai.settings:url',
|
||||
];
|
||||
$form['model'] = [
|
||||
'#type' => 'textfield',
|
||||
'#title' => $this->t('Model'),
|
||||
'#required' => TRUE,
|
||||
'#config_target' => 'materio_ai.settings:model',
|
||||
];
|
||||
$form['dimension'] = [
|
||||
'#type' => 'number',
|
||||
'#title' => $this->t('Vector dimension'),
|
||||
'#min' => 1,
|
||||
'#required' => TRUE,
|
||||
'#config_target' => 'materio_ai.settings:dimension',
|
||||
];
|
||||
$form['timeout'] = [
|
||||
'#type' => 'number',
|
||||
'#title' => $this->t('Timeout (seconds)'),
|
||||
'#min' => 1,
|
||||
'#required' => TRUE,
|
||||
'#config_target' => 'materio_ai.settings:timeout',
|
||||
];
|
||||
$form['num_thread'] = [
|
||||
'#type' => 'number',
|
||||
'#title' => $this->t('Threads per call'),
|
||||
'#description' => $this->t('Sent as options.num_thread: Ollama does not see the CPU limit of its container.'),
|
||||
'#min' => 1,
|
||||
'#required' => TRUE,
|
||||
'#config_target' => 'materio_ai.settings:num_thread',
|
||||
];
|
||||
return parent::buildForm($form, $form_state);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Plugin\QueueWorker;
|
||||
|
||||
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
|
||||
use Drupal\Core\Queue\Attribute\QueueWorker;
|
||||
use Drupal\Core\Queue\QueueWorkerBase;
|
||||
use Drupal\Core\Queue\SuspendQueueException;
|
||||
use Drupal\Core\StringTranslation\TranslatableMarkup;
|
||||
use Drupal\materio_ai\Exception\EmbeddingException;
|
||||
use Drupal\materio_ai\Vectorizer;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\DependencyInjection\ContainerInterface;
|
||||
|
||||
/**
|
||||
* Vectorizes the nodes queued on save, during cron.
|
||||
*
|
||||
* When the model is unreachable, the queue is suspended: the item stays in the
|
||||
* queue and is retried at the next cron run.
|
||||
*/
|
||||
#[QueueWorker(
|
||||
id: Vectorizer::QUEUE,
|
||||
title: new TranslatableMarkup('materiO AI vectorization'),
|
||||
cron: ['time' => 60],
|
||||
)]
|
||||
final class VectorizeNode extends QueueWorkerBase implements ContainerFactoryPluginInterface {
|
||||
|
||||
public function __construct(
|
||||
array $configuration,
|
||||
string $plugin_id,
|
||||
mixed $plugin_definition,
|
||||
private readonly Vectorizer $vectorizer,
|
||||
private readonly LoggerInterface $logger,
|
||||
) {
|
||||
parent::__construct($configuration, $plugin_id, $plugin_definition);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition): self {
|
||||
return new self(
|
||||
$configuration,
|
||||
$plugin_id,
|
||||
$plugin_definition,
|
||||
$container->get('materio_ai.vectorizer'),
|
||||
$container->get('logger.channel.materio_ai'),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function processItem($data): void {
|
||||
if (!$this->vectorizer->isEnabled()) {
|
||||
// Switched off after the item was queued: keep it for later.
|
||||
throw new SuspendQueueException('materiO AI is disabled.');
|
||||
}
|
||||
$nid = (int) ($data['nid'] ?? 0);
|
||||
try {
|
||||
$this->vectorizer->vectorize($nid);
|
||||
}
|
||||
catch (EmbeddingException $e) {
|
||||
$this->logger->warning('Vectorization of node @nid postponed: @message', ['@nid' => $nid, '@message' => $e->getMessage()]);
|
||||
throw new SuspendQueueException($e->getMessage(), 0, $e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Search;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\Core\Entity\EntityTypeManagerInterface;
|
||||
use Drupal\Core\Language\LanguageManagerInterface;
|
||||
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
|
||||
use Drupal\search_api\Utility\Utility;
|
||||
|
||||
/**
|
||||
* Finds the nodes closest in meaning to a phrase (kNN in Solr).
|
||||
*
|
||||
* Runs a normal Search API query on the database index, tagged so that
|
||||
* KnnQuerySubscriber turns it, and only it, into a {!knn} query. Search API
|
||||
* processors still add their conditions (content_access: node grants and
|
||||
* status for the current user; language), which Solr applies as pre-filters
|
||||
* of the kNN search (Solr 9.1+): the vectors also cover nodes the user may
|
||||
* not see, only these filters exclude them.
|
||||
*/
|
||||
final class NeighbourSearch {
|
||||
|
||||
/**
|
||||
* Tag of the Search API query turned into a kNN query.
|
||||
*/
|
||||
public const TAG = 'materio_ai_knn';
|
||||
|
||||
/**
|
||||
* Query option holding the query vector.
|
||||
*/
|
||||
public const OPTION_VECTOR = 'materio_ai_knn_vector';
|
||||
|
||||
/**
|
||||
* Search API index holding the vectors (I2).
|
||||
*/
|
||||
public const INDEX = 'database';
|
||||
|
||||
public function __construct(
|
||||
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||
private readonly LanguageManagerInterface $languageManager,
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
private readonly EmbeddingClientInterface $client,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Nodes closest to a phrase, best first.
|
||||
*
|
||||
* @param string $phrase
|
||||
* The phrase typed by the user.
|
||||
* @param string|null $langcode
|
||||
* Language of the results; the current language by default.
|
||||
* @param int $limit
|
||||
* Number of neighbours (topK).
|
||||
*
|
||||
* @return array<int, float>
|
||||
* Scores keyed by nid, best first. Solr score of a cosine kNN query:
|
||||
* (1 + cosine) / 2.
|
||||
*
|
||||
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||
* When the model fails.
|
||||
* @throws \Drupal\search_api\SearchApiException
|
||||
* When Solr fails.
|
||||
*/
|
||||
public function search(string $phrase, ?string $langcode = NULL, int $limit = 100): array {
|
||||
return $this->searchVector($this->client->embed($phrase), $langcode, $limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Nodes closest to a vector, best first (see search()).
|
||||
*
|
||||
* @param float[] $vector
|
||||
* Query vector, of the configured dimension.
|
||||
*
|
||||
* @return array<int, float>
|
||||
* Scores keyed by nid, best first.
|
||||
*/
|
||||
public function searchVector(array $vector, ?string $langcode = NULL, int $limit = 100): array {
|
||||
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
|
||||
if (count($vector) !== $dimension || $limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
$index = $this->entityTypeManager->getStorage('search_api_index')->load(self::INDEX);
|
||||
$query = $index->query();
|
||||
$query->setLanguages([$langcode ?? $this->languageManager->getCurrentLanguage()->getId()]);
|
||||
$query->range(0, $limit);
|
||||
$query->sort('search_api_relevance', 'DESC');
|
||||
$query->setOption(self::OPTION_VECTOR, array_values($vector));
|
||||
// Only the item IDs are needed.
|
||||
$query->setOption('search_api_retrieved_field_values', ['search_api_id' => 'search_api_id']);
|
||||
$query->addTag(self::TAG);
|
||||
|
||||
$scores = [];
|
||||
foreach ($query->execute() as $item_id => $item) {
|
||||
[, $raw_id] = Utility::splitCombinedId($item_id);
|
||||
$nid = (int) explode(':', $raw_id, 2)[0];
|
||||
$scores[$nid] ??= (float) $item->getScore();
|
||||
}
|
||||
return $scores;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Solr;
|
||||
|
||||
/**
|
||||
* Names and schema snippets of the dense vector field in Solr.
|
||||
*
|
||||
* One field type and one dynamic field per dimension: a model of another
|
||||
* dimension gets its own field, without touching the existing one.
|
||||
*/
|
||||
final class VectorField {
|
||||
|
||||
/**
|
||||
* Field type name for a dimension, e.g. knn_vector_1024.
|
||||
*/
|
||||
public static function typeName(int $dimension): string {
|
||||
return 'knn_vector_' . $dimension;
|
||||
}
|
||||
|
||||
/**
|
||||
* Solr field written to each document, e.g. knn1024_vector.
|
||||
*/
|
||||
public static function fieldName(int $dimension): string {
|
||||
return 'knn' . $dimension . '_vector';
|
||||
}
|
||||
|
||||
/**
|
||||
* <fieldType> for schema_extra_types.xml.
|
||||
*
|
||||
* Cosine: no assumption on the norm of the vectors sent by the model.
|
||||
*/
|
||||
public static function typeXml(int $dimension): string {
|
||||
return sprintf(
|
||||
"<fieldType name=\"%s\" class=\"solr.DenseVectorField\" vectorDimension=\"%d\" similarityFunction=\"cosine\" knnAlgorithm=\"hnsw\" />\n",
|
||||
self::typeName($dimension),
|
||||
$dimension,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* <dynamicField> for schema_extra_fields.xml.
|
||||
*
|
||||
* Not stored: the vectors live in the materio_ai_embedding table.
|
||||
*/
|
||||
public static function dynamicFieldXml(int $dimension): string {
|
||||
return sprintf(
|
||||
"<dynamicField name=\"knn%d_*\" type=\"%s\" indexed=\"true\" stored=\"false\" multiValued=\"false\" />\n",
|
||||
$dimension,
|
||||
self::typeName($dimension),
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Storage;
|
||||
|
||||
use Drupal\Component\Datetime\TimeInterface;
|
||||
use Drupal\Core\Database\Connection;
|
||||
|
||||
/**
|
||||
* Reads and writes the vectors of the materio_ai_embedding table.
|
||||
*
|
||||
* Vectors are stored as little-endian 32-bit floats: 4 KB for 1 024 values.
|
||||
*/
|
||||
final class EmbeddingStorage {
|
||||
|
||||
private const TABLE = 'materio_ai_embedding';
|
||||
|
||||
public function __construct(
|
||||
private readonly Connection $database,
|
||||
private readonly TimeInterface $time,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Inserts or replaces the vector of a node translation.
|
||||
*
|
||||
* @param float[] $vector
|
||||
* The vector.
|
||||
*/
|
||||
public function save(int $nid, string $langcode, string $model, string $text_hash, array $vector): void {
|
||||
$this->database->merge(self::TABLE)
|
||||
->keys(['nid' => $nid, 'langcode' => $langcode])
|
||||
->fields([
|
||||
'model' => $model,
|
||||
'text_hash' => $text_hash,
|
||||
'vector' => self::pack($vector),
|
||||
'changed' => $this->time->getRequestTime(),
|
||||
])
|
||||
->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the stored row of a node translation, with the vector unpacked.
|
||||
*
|
||||
* @return array{nid: int, langcode: string, model: string, text_hash: string, vector: float[], changed: int}|null
|
||||
* The row, or NULL when the translation has no vector.
|
||||
*/
|
||||
public function load(int $nid, string $langcode): ?array {
|
||||
$row = $this->database->select(self::TABLE, 'e')
|
||||
->fields('e')
|
||||
->condition('nid', $nid)
|
||||
->condition('langcode', $langcode)
|
||||
->execute()
|
||||
->fetchAssoc();
|
||||
return $row ? self::row($row) : NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the vectors of several nodes computed by one model.
|
||||
*
|
||||
* One query for a whole indexing batch.
|
||||
*
|
||||
* @param int[] $nids
|
||||
* Node IDs.
|
||||
*
|
||||
* @return array<int, array<string, float[]>>
|
||||
* Vectors keyed by nid, then langcode.
|
||||
*/
|
||||
public function loadVectors(array $nids, string $model): array {
|
||||
if (!$nids) {
|
||||
return [];
|
||||
}
|
||||
$result = $this->database->select(self::TABLE, 'e')
|
||||
->fields('e', ['nid', 'langcode', 'vector'])
|
||||
->condition('nid', array_values(array_unique($nids)), 'IN')
|
||||
->condition('model', $model)
|
||||
->execute();
|
||||
$vectors = [];
|
||||
foreach ($result as $row) {
|
||||
$vectors[(int) $row->nid][$row->langcode] = self::unpack($row->vector);
|
||||
}
|
||||
return $vectors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the text hashes of a node, keyed by langcode.
|
||||
*
|
||||
* Lets the queue skip texts that did not change, without loading vectors.
|
||||
*
|
||||
* @return array<string, array{model: string, text_hash: string}>
|
||||
* The hashes and models.
|
||||
*/
|
||||
public function loadHashes(int $nid): array {
|
||||
return $this->database->select(self::TABLE, 'e')
|
||||
->fields('e', ['langcode', 'model', 'text_hash'])
|
||||
->condition('nid', $nid)
|
||||
->execute()
|
||||
->fetchAllAssoc('langcode', \PDO::FETCH_ASSOC);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the vectors of a node, or of one of its translations.
|
||||
*/
|
||||
public function delete(int $nid, ?string $langcode = NULL): int {
|
||||
$query = $this->database->delete(self::TABLE)->condition('nid', $nid);
|
||||
if ($langcode !== NULL) {
|
||||
$query->condition('langcode', $langcode);
|
||||
}
|
||||
return (int) $query->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts the stored vectors, optionally for one model.
|
||||
*/
|
||||
public function count(?string $model = NULL): int {
|
||||
$query = $this->database->select(self::TABLE, 'e');
|
||||
if ($model !== NULL) {
|
||||
$query->condition('model', $model);
|
||||
}
|
||||
return (int) $query->countQuery()->execute()->fetchField();
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash of a text, as stored in text_hash.
|
||||
*/
|
||||
public static function hash(string $text): string {
|
||||
return hash('sha256', $text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Packs a vector as little-endian 32-bit floats.
|
||||
*
|
||||
* @param float[] $vector
|
||||
* The vector.
|
||||
*/
|
||||
public static function pack(array $vector): string {
|
||||
return pack('g*', ...array_values($vector));
|
||||
}
|
||||
|
||||
/**
|
||||
* Unpacks a vector stored by pack().
|
||||
*
|
||||
* @return float[]
|
||||
* The vector, keys starting at 0.
|
||||
*/
|
||||
public static function unpack(string $blob): array {
|
||||
return array_values(unpack('g*', $blob));
|
||||
}
|
||||
|
||||
/**
|
||||
* Casts a database row and unpacks its vector.
|
||||
*/
|
||||
private static function row(array $row): array {
|
||||
return [
|
||||
'nid' => (int) $row['nid'],
|
||||
'langcode' => $row['langcode'],
|
||||
'model' => $row['model'],
|
||||
'text_hash' => $row['text_hash'],
|
||||
'vector' => self::unpack($row['vector']),
|
||||
'changed' => (int) $row['changed'],
|
||||
];
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\Core\Entity\EntityTypeManagerInterface;
|
||||
use Drupal\Core\Queue\QueueFactory;
|
||||
use Drupal\materio_ai\Corpus\CorpusBuilder;
|
||||
use Drupal\materio_ai\Embedding\EmbeddingClientInterface;
|
||||
use Drupal\materio_ai\Storage\EmbeddingStorage;
|
||||
use Drupal\node\NodeInterface;
|
||||
use Drupal\search_api\Plugin\search_api\datasource\ContentEntity;
|
||||
use Drupal\taxonomy\TermInterface;
|
||||
|
||||
/**
|
||||
* Keeps the vectors of a node in step with its texts.
|
||||
*
|
||||
* Used by the vectorization queue (cron) and by the full vectorization
|
||||
* command. The model is only called for texts whose hash or model changed.
|
||||
*/
|
||||
final class Vectorizer {
|
||||
|
||||
/**
|
||||
* Name of the vectorization queue.
|
||||
*/
|
||||
public const QUEUE = 'materio_ai_vectorize';
|
||||
|
||||
/**
|
||||
* Whether Search API indexes the marked items at the end of the request.
|
||||
*
|
||||
* FALSE during the full vectorization: items are only marked, a full
|
||||
* reindex follows (drush sapi-i).
|
||||
*/
|
||||
private bool $directIndexing = TRUE;
|
||||
|
||||
public function __construct(
|
||||
private readonly EntityTypeManagerInterface $entityTypeManager,
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
private readonly QueueFactory $queueFactory,
|
||||
private readonly CorpusBuilder $corpusBuilder,
|
||||
private readonly EmbeddingClientInterface $client,
|
||||
private readonly EmbeddingStorage $storage,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Tells whether vectorization is switched on (materio_ai.settings:enabled).
|
||||
*/
|
||||
public function isEnabled(): bool {
|
||||
return (bool) $this->configFactory->get('materio_ai.settings')->get('enabled');
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues a node for vectorization, when it is a vectorized type.
|
||||
*/
|
||||
public function enqueue(NodeInterface $node): void {
|
||||
if ($this->isEnabled() && in_array($node->bundle(), CorpusBuilder::BUNDLES, TRUE)) {
|
||||
$this->queueFactory->get(self::QUEUE)->createItem(['nid' => (int) $node->id()]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Node fields whose referenced terms go into the text (CorpusBuilder).
|
||||
*/
|
||||
public const TERM_FIELDS = ['field_tags', 'field_thesaurus', 'field_manufacturer', 'field_distributor'];
|
||||
|
||||
/**
|
||||
* Queues the nodes that use a term, when its text changed (V7).
|
||||
*
|
||||
* A renamed term, a new synonym or a company that moved changes the text of
|
||||
* every node referencing it, in each language. Unchanged languages are then
|
||||
* skipped by their hash, without calling the model.
|
||||
*
|
||||
* @return int
|
||||
* Number of queued nodes.
|
||||
*/
|
||||
public function enqueueForTerm(TermInterface $term, bool $force = FALSE): int {
|
||||
if (!$this->isEnabled() || (!$force && !$this->termTextChanged($term))) {
|
||||
return 0;
|
||||
}
|
||||
$query = $this->entityTypeManager->getStorage('node')->getQuery()
|
||||
->accessCheck(FALSE)
|
||||
->condition('type', CorpusBuilder::BUNDLES, 'IN');
|
||||
$or = $query->orConditionGroup();
|
||||
foreach (self::TERM_FIELDS as $field) {
|
||||
$or->condition($field, $term->id());
|
||||
}
|
||||
$nids = $query->condition($or)->execute();
|
||||
$queue = $this->queueFactory->get(self::QUEUE);
|
||||
foreach ($nids as $nid) {
|
||||
$queue->createItem(['nid' => (int) $nid]);
|
||||
}
|
||||
return count($nids);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells whether what the corpus reads from a term changed on save.
|
||||
*
|
||||
* Compares, for every translation, the name, the synonyms and the country
|
||||
* of the public address with the original term.
|
||||
*/
|
||||
private function termTextChanged(TermInterface $term): bool {
|
||||
$original = method_exists($term, 'getOriginal') ? $term->getOriginal() : ($term->original ?? NULL);
|
||||
if (!$original instanceof TermInterface) {
|
||||
return TRUE;
|
||||
}
|
||||
$langcodes = array_unique(array_merge(array_keys($term->getTranslationLanguages()), array_keys($original->getTranslationLanguages())));
|
||||
foreach ($langcodes as $langcode) {
|
||||
if (!$term->hasTranslation($langcode) || !$original->hasTranslation($langcode)) {
|
||||
return TRUE;
|
||||
}
|
||||
if (self::termText($term->getTranslation($langcode)) !== self::termText($original->getTranslation($langcode))) {
|
||||
return TRUE;
|
||||
}
|
||||
}
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
/**
|
||||
* What the corpus reads from one term translation.
|
||||
*/
|
||||
private static function termText(TermInterface $term): array {
|
||||
$text = [$term->label()];
|
||||
if ($term->hasField('field_synonyms')) {
|
||||
$text[] = array_column($term->get('field_synonyms')->getValue(), 'value');
|
||||
}
|
||||
if ($term->hasField('field_public_address')) {
|
||||
$text[] = $term->get('field_public_address')->first()?->getCountryCode();
|
||||
}
|
||||
return $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the vectors of a node.
|
||||
*
|
||||
* Deletes the vectors of translations that left the corpus (node deleted,
|
||||
* unpublished, translation removed), then vectorizes new or changed texts.
|
||||
*
|
||||
* @return array{embedded: int, unchanged: int, deleted: int, marked: int}
|
||||
* What was done, per translation; marked: items marked for reindexing.
|
||||
*
|
||||
* @throws \Drupal\materio_ai\Exception\EmbeddingException
|
||||
* When the model fails: vectors already saved for other translations stay.
|
||||
*/
|
||||
public function vectorize(int $nid): array {
|
||||
$result = ['embedded' => 0, 'unchanged' => 0, 'deleted' => 0, 'marked' => 0];
|
||||
$node = $this->entityTypeManager->getStorage('node')->load($nid);
|
||||
$texts = $node instanceof NodeInterface ? $this->corpusBuilder->buildAll($node) : [];
|
||||
$stored = $this->storage->loadHashes($nid);
|
||||
// Translations whose vector changed: their Solr document is outdated.
|
||||
$changed = [];
|
||||
|
||||
try {
|
||||
foreach (array_diff_key($stored, $texts) as $langcode => $row) {
|
||||
if ($this->storage->delete($nid, (string) $langcode)) {
|
||||
$result['deleted']++;
|
||||
$changed[] = (string) $langcode;
|
||||
}
|
||||
}
|
||||
|
||||
$model = $this->client->getModel();
|
||||
foreach ($texts as $langcode => $text) {
|
||||
$hash = EmbeddingStorage::hash($text);
|
||||
if (isset($stored[$langcode]) && $stored[$langcode]['text_hash'] === $hash && $stored[$langcode]['model'] === $model) {
|
||||
$result['unchanged']++;
|
||||
continue;
|
||||
}
|
||||
$this->storage->save($nid, $langcode, $model, $hash, $this->client->embed($text));
|
||||
$result['embedded']++;
|
||||
$changed[] = (string) $langcode;
|
||||
}
|
||||
}
|
||||
finally {
|
||||
// Also when the model fails half-way: vectors already saved are marked.
|
||||
if ($changed && $node instanceof NodeInterface) {
|
||||
$result['marked'] = $this->markForReindex($node, $changed);
|
||||
}
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Switches indexing at the end of the request on or off (see property).
|
||||
*/
|
||||
public function setDirectIndexing(bool $direct): void {
|
||||
$this->directIndexing = $direct;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks translations of a node for reindexing in the indexes holding it.
|
||||
*
|
||||
* The Solr document gets the new vector at the next indexing (I2): at the
|
||||
* end of the request with index_directly, otherwise at the next cron.
|
||||
* A deleted node or translation is removed from the index by Search API.
|
||||
*
|
||||
* @param string[] $langcodes
|
||||
* Translations whose vector changed.
|
||||
*
|
||||
* @return int
|
||||
* Number of items marked, all indexes together.
|
||||
*/
|
||||
private function markForReindex(NodeInterface $node, array $langcodes): int {
|
||||
$ids = [];
|
||||
foreach (array_unique($langcodes) as $langcode) {
|
||||
if ($node->hasTranslation($langcode)) {
|
||||
$ids[] = $node->id() . ':' . $langcode;
|
||||
}
|
||||
}
|
||||
if (!$ids) {
|
||||
return 0;
|
||||
}
|
||||
$marked = 0;
|
||||
foreach (ContentEntity::getIndexesForEntity($node) as $index) {
|
||||
if (!$this->directIndexing) {
|
||||
$index->startBatchTracking();
|
||||
}
|
||||
try {
|
||||
$index->trackItemsUpdated('entity:node', $ids);
|
||||
}
|
||||
finally {
|
||||
if (!$this->directIndexing) {
|
||||
$index->stopBatchTracking();
|
||||
}
|
||||
}
|
||||
$marked += count($ids);
|
||||
}
|
||||
return $marked;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -6,8 +6,9 @@ namespace Drupal\materio_sapi\Controller;
|
||||
use Drupal\Core\Controller\ControllerBase;
|
||||
use Symfony\Component\HttpFoundation\JsonResponse;
|
||||
use Symfony\Component\HttpFoundation\Request;
|
||||
use Drupal\Component\Utility\Tags;
|
||||
use Drupal\Component\Utility\Unicode;
|
||||
use Drupal\materio_sapi\KeywordText;
|
||||
use Drupal\materio_sapi\RankFusion;
|
||||
use Drupal\search_api\Entity\Index;
|
||||
|
||||
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
|
||||
@@ -53,153 +54,223 @@ class Base extends ControllerBase {
|
||||
protected $phrase_query, $and_query, $or_query, $query, $count_query, $count, $exactematch_count;
|
||||
|
||||
/**
|
||||
* Filter and term values.
|
||||
* Term values (autocomplete choice) and items.
|
||||
*
|
||||
* @var mixed
|
||||
*/
|
||||
protected $filters, $terms, $items;
|
||||
protected $terms, $items;
|
||||
|
||||
private $limit = 15;
|
||||
private $offset = 0;
|
||||
|
||||
/**
|
||||
* Longest phrase kept, in characters (H1, 150 decided on 17/11). The longest
|
||||
* test phrase (S5) has 89 characters; beyond the limit the phrase is cut,
|
||||
* not refused.
|
||||
*/
|
||||
public const MAX_PHRASE_LENGTH = 150;
|
||||
|
||||
/**
|
||||
* The phrase typed by the user, whole and case kept, for the model (H5).
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $phrase = '';
|
||||
|
||||
/**
|
||||
* Material references found in the phrase (W0117…), uppercase (H4).
|
||||
*
|
||||
* @var string[]
|
||||
*/
|
||||
protected $references = [];
|
||||
|
||||
/**
|
||||
* Pattern of a material reference: family letter and four digits.
|
||||
*/
|
||||
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
|
||||
|
||||
/**
|
||||
* Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6.
|
||||
*/
|
||||
public const KNN_LIMIT = 200;
|
||||
|
||||
/**
|
||||
* Number of neighbours found, for the response tuning of H6.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
protected $knn_count = 0;
|
||||
|
||||
/**
|
||||
* The reference query, run before the others.
|
||||
*
|
||||
* @var \Drupal\search_api\Query\QueryInterface|null
|
||||
*/
|
||||
protected $reference_query;
|
||||
|
||||
/**
|
||||
* Most meaningful words for which the phrase and AND queries run (H2, a).
|
||||
*
|
||||
* Beyond, they find nothing (no node holds every word of a long phrase) and
|
||||
* the AND query may exceed maxClauseCount: only the OR query runs.
|
||||
*/
|
||||
public const SHORT_QUERY_WORDS = 6;
|
||||
|
||||
/**
|
||||
* Runs the keyword queries and merges their results, exact matches first.
|
||||
*
|
||||
* The keys are the phrase without stop words (H2, b): the custom types of
|
||||
* materio_sapi keep them, « de » alone matched 2 236 nodes.
|
||||
*/
|
||||
private function sapiQuery(){
|
||||
// https://www.drupal.org/docs/8/modules/search-api/developer-documentation/executing-a-search-in-code
|
||||
// https://www.hashbangcode.com/article/drupal-8-date-search-boosting-search-api-and-solr-search
|
||||
// https://kgaut.net/blog/2018/drupal-8-search-api-effectuer-une-requete-dans-le-code.html
|
||||
|
||||
|
||||
// Otherwise, if you use Solr, you can also set the parse mode to “Direct query”
|
||||
// and use something like 'tm_name:alex AND tm_surname:john' as the keywords (field names may vary).
|
||||
// That will also ensure scoring works for the keywords (the other way just adds filters, which don’t affect scoring).
|
||||
// https://www.drupal.org/project/search_api/issues/3049097
|
||||
|
||||
|
||||
|
||||
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
|
||||
|
||||
$this->index = Index::load('database');
|
||||
|
||||
// References first, as exact matches, whatever their case (H4, B2).
|
||||
preg_match_all(self::REFERENCE_PATTERN, $this->phrase, $matches);
|
||||
$this->references = array_values(array_unique(array_map('mb_strtoupper', $matches[0])));
|
||||
$exact = $this->references ? $this->referenceQuery($this->references, $lang) : [];
|
||||
if ($this->isReferenceSearch()) {
|
||||
$this->exactematch_count = count($exact);
|
||||
$this->results = ['uuids' => array_column($exact, 1), 'nids' => array_column($exact, 0)];
|
||||
return;
|
||||
}
|
||||
|
||||
// Keys without the references (« bois R0208 » searches « bois »).
|
||||
$words = KeywordText::words(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? $this->phrase, $lang);
|
||||
// Only stop words: search the phrase as typed, as before.
|
||||
$keys = $words ? implode(' ', $words) : implode(' ', $this->keys);
|
||||
|
||||
if (count($words) <= self::SHORT_QUERY_WORDS) {
|
||||
// Exact material names (like "wood-skin"), then all words.
|
||||
$exact += $this->phraseQuery($keys, $lang);
|
||||
$exact += $this->andQuery($keys, $lang);
|
||||
}
|
||||
$this->exactematch_count = count($exact);
|
||||
$or = $this->orQuery($keys, $lang);
|
||||
|
||||
// Exact matches first, then the OR query and the neighbours merged (H5).
|
||||
$neighbours = $this->neighbours($lang);
|
||||
$this->knn_count = count($neighbours);
|
||||
$nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]);
|
||||
|
||||
$uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or)));
|
||||
$this->results = [
|
||||
'uuids' => [],
|
||||
'nids' => []
|
||||
'uuids' => array_map(static fn ($nid) => $uuids[$nid], $nids),
|
||||
'nids' => $nids,
|
||||
];
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// ,---.| ,---.
|
||||
// |---'|---.,---.,---.,---.,---. | |. .,---.,---., .
|
||||
// | | || ,---|`---.|---' | || ||---'| | |
|
||||
// ` ` '` `---^`---'`---' `---\`---'`---'` `---|
|
||||
// `---'
|
||||
// (to match exact materials names (like "wood-skin"))
|
||||
$this->phrase_query = $this->index->query(['offset'=>0,'limit'=>10000]);
|
||||
// set parse mode and conjunction
|
||||
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('phrase');
|
||||
$parse_mode->setConjunction('AND');
|
||||
$this->phrase_query->setParseMode($parse_mode);
|
||||
// Set fulltext search keywords and fields.
|
||||
if ($this->keys) {
|
||||
$this->phrase_query->keys(implode(' ', $this->keys));
|
||||
}
|
||||
|
||||
/**
|
||||
* Nodes closest in meaning to the phrase (materio_ai, Solr kNN).
|
||||
*
|
||||
* Not for a term chosen in the autocomplete (direct search) nor without
|
||||
* the materio_ai service. On a model or Solr error, the keyword results
|
||||
* stand alone (delay and cache in H8).
|
||||
*
|
||||
* @return array<int, float>
|
||||
* Scores keyed by nid, best first.
|
||||
*/
|
||||
private function neighbours(string $lang): array {
|
||||
if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) {
|
||||
return [];
|
||||
}
|
||||
// References are keywords: they mean nothing to the model.
|
||||
$phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? '');
|
||||
if ($phrase === '') {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT);
|
||||
}
|
||||
catch (\Throwable $e) {
|
||||
\Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* UUIDs of nodes found only by the neighbour search.
|
||||
*
|
||||
* @param int[] $nids
|
||||
* Node IDs.
|
||||
*
|
||||
* @return array<int, string>
|
||||
* UUIDs keyed by nid.
|
||||
*/
|
||||
private function uuids(array $nids): array {
|
||||
if (!$nids) {
|
||||
return [];
|
||||
}
|
||||
return \Drupal::database()->select('node', 'n')
|
||||
->fields('n', ['nid', 'uuid'])
|
||||
->condition('nid', array_values($nids), 'IN')
|
||||
->execute()
|
||||
->fetchAllKeyed();
|
||||
}
|
||||
|
||||
/**
|
||||
* Nodes whose reference is one of the given ones.
|
||||
*
|
||||
* field_reference is case-sensitive: references are uppercased (B2).
|
||||
*
|
||||
* @param string[] $references
|
||||
* Uppercase references.
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid, in result order.
|
||||
*/
|
||||
private function referenceQuery(array $references, string $lang): array {
|
||||
$this->reference_query = $this->keywordQuery('terms', 'OR', '', $lang, 'materio_sapi_search_reference_query');
|
||||
$conditions = $this->reference_query->createConditionGroup('OR');
|
||||
foreach ($references as $reference) {
|
||||
$conditions->addCondition('field_reference', $reference);
|
||||
}
|
||||
$this->reference_query->addConditionGroup($conditions);
|
||||
return $this->collect($this->reference_query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the phrase is only references: no need for the model (H4, H5).
|
||||
*/
|
||||
public function isReferenceSearch(): bool {
|
||||
return $this->references && trim(preg_replace(self::REFERENCE_PATTERN, '', $this->phrase) ?? '', " \t,;") === '';
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole keys in the title.
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid, in result order.
|
||||
*/
|
||||
private function phraseQuery(string $keys, string $lang): array {
|
||||
$this->phrase_query = $this->keywordQuery('phrase', 'AND', $keys, $lang, 'materio_sapi_search_phrase_query');
|
||||
$this->phrase_query->setFulltextFields(['title']);
|
||||
|
||||
// Restrict the search to specific languages.
|
||||
$this->phrase_query->setLanguages([$lang]);
|
||||
return $this->collect($this->phrase_query);
|
||||
}
|
||||
|
||||
// Do paging.
|
||||
// $this->and_query->range($this->offset, $this->limit);
|
||||
// retrieve all results
|
||||
// $this->and_query->range(0, -1);
|
||||
/**
|
||||
* Every word in the thesaurus and tag fields, or the chosen terms.
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid, in result order.
|
||||
*/
|
||||
private function andQuery(string $keys, string $lang): array {
|
||||
$this->and_query = $this->keywordQuery('terms', 'AND', $keys, $lang, 'materio_sapi_search_and_query');
|
||||
|
||||
// Add sorting.
|
||||
$this->phrase_query->sort('search_api_relevance', 'DESC');
|
||||
// Tiebreaker: among equally relevant matches, show the most recently
|
||||
// modified first (Solr otherwise falls back to internal doc order, i.e.
|
||||
// indexation order, which puts the oldest materiaux first).
|
||||
$this->phrase_query->sort('changed', 'DESC');
|
||||
|
||||
// Set one or more tags for the query.
|
||||
// @see hook_search_api_query_TAG_alter()
|
||||
// @see hook_search_api_results_TAG_alter()
|
||||
$this->phrase_query->addTag('materio_sapi_search_phrase_query');
|
||||
|
||||
$phrase_results = $this->phrase_query->execute();
|
||||
|
||||
foreach ($phrase_results as $result) {
|
||||
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
|
||||
$this->results['nids'][] = $result->getField('nid')->getValues()[0];
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ,---. | ,---.
|
||||
// |---|,---.,---| | |. .,---.,---., .
|
||||
// | || || | | || ||---'| | |
|
||||
// ` '` '`---' `---\`---'`---'` `---|
|
||||
// `---'
|
||||
$this->and_query = $this->index->query(['offset'=>0,'limit'=>10000]);
|
||||
// set parse mode and conjunction
|
||||
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('terms');
|
||||
$parse_mode->setConjunction('AND');
|
||||
$this->and_query->setParseMode($parse_mode);
|
||||
// Set fulltext search keywords and fields.
|
||||
if ($this->keys) {
|
||||
$this->and_query->keys(implode(' ', $this->keys));
|
||||
}
|
||||
|
||||
// in case of term id provided restrict the keys to taxo fields
|
||||
// A term chosen in the autocomplete restricts to the taxonomy fields.
|
||||
if ($this->terms && is_array($this->terms) && count($this->terms)) {
|
||||
$term_conditions = $this->and_query->createConditionGroup('OR');
|
||||
// $term = (int) $this->term;
|
||||
foreach ($this->terms as $term) {
|
||||
$tid = $term->value;
|
||||
foreach (['tag_tid', 'thesaurus_tid'] as $field) {
|
||||
$term_conditions->addCondition($field, (int) $tid);
|
||||
$term_conditions->addCondition($field, (int) $term->value);
|
||||
}
|
||||
}
|
||||
$this->and_query->addConditionGroup($term_conditions);
|
||||
|
||||
// INSTEAD TRY TO BOOST THE TAG AND THESAURUS FIELDS
|
||||
// foreach ($taxoSolrFieldsName as $fname) {
|
||||
// // $solarium_query->addParam('bf', "recip(abs(ms(NOW,{$solrField})),3.16e-11,10,0.1)");
|
||||
// $bfparam = "if(gt(termfreq({$fname},{$this->term}),0),^21,0)";
|
||||
// $this->or_query->addParam('bf', $bfparam);
|
||||
// }
|
||||
// look @ materio_sapi_search_api_solr_query_alter in materio_sapi.module
|
||||
// $this->or_query->setOption('termid', $this->term);
|
||||
}
|
||||
|
||||
if ($this->filters) {
|
||||
// FILTERS
|
||||
$filters_conditions = $this->and_query->createConditionGroup('AND');
|
||||
foreach ($this->filters as $filter) {
|
||||
$filter = (int) $filter;
|
||||
foreach (['thesaurus_tid'] as $field) { // 'tag_tid',
|
||||
$filters_conditions->addCondition($field, $filter);
|
||||
}
|
||||
}
|
||||
$this->and_query->addConditionGroup($filters_conditions);
|
||||
|
||||
if(!$this->keys) {
|
||||
// if no keys but filters switch query to direct and add wildcard solr keys *:*
|
||||
$direct_and_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('direct');
|
||||
$direct_and_parse_mode->setConjunction('AND');
|
||||
$this->and_query->setParseMode($direct_and_parse_mode);
|
||||
// $this->and_query->keys('*:*');
|
||||
}
|
||||
}
|
||||
// else{
|
||||
$fulltextFields = [];
|
||||
// Recherche uniquement sur les champ thésaurus et tag
|
||||
$fulltextFields += [
|
||||
// Thesaurus and tag fields only.
|
||||
$this->and_query->setFulltextFields([
|
||||
'thesaurus_name_0',
|
||||
'thesaurus_synonyms_0',
|
||||
'thesaurus_name_1',
|
||||
@@ -213,144 +284,67 @@ class Base extends ControllerBase {
|
||||
'thesaurus_name',
|
||||
'thesaurus_synonyms',
|
||||
'tag_name',
|
||||
'tag_synonyms'
|
||||
];
|
||||
if(count($fulltextFields)){
|
||||
$this->and_query->setFulltextFields($fulltextFields);
|
||||
'tag_synonyms',
|
||||
]);
|
||||
return $this->collect($this->and_query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Any word, in every fulltext field.
|
||||
*
|
||||
* References are no longer a condition here (H4): it restricted the OR
|
||||
* results to the referenced nodes (« bois R0208 » found nothing).
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid, in result order.
|
||||
*/
|
||||
private function orQuery(string $keys, string $lang): array {
|
||||
// Terms mode, not direct: Search API escapes the words, so punctuation
|
||||
// and Solr syntax typed in a phrase (: ( " / \\ ~) are plain text (H3).
|
||||
// Hyphenated words as separate words, as the direct mode did: in terms mode
|
||||
// « bio-based » would be a quoted phrase (the phrase query keeps it whole).
|
||||
$this->or_query = $this->keywordQuery('terms', 'OR', preg_replace('/(?<=\p{L})-(?=\p{L})/u', ' ', $keys), $lang, 'materio_sapi_search_or_query');
|
||||
return $this->collect($this->or_query);
|
||||
}
|
||||
|
||||
/**
|
||||
* A keyword query on the database index, not executed yet.
|
||||
*
|
||||
* @return \Drupal\search_api\Query\QueryInterface
|
||||
* The query.
|
||||
*/
|
||||
private function keywordQuery(string $parse_mode_id, string $conjunction, string $keys, string $lang, string $tag) {
|
||||
$query = $this->index->query(['offset' => 0, 'limit' => 10000]);
|
||||
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')->createInstance($parse_mode_id);
|
||||
$parse_mode->setConjunction($conjunction);
|
||||
$query->setParseMode($parse_mode);
|
||||
if ($keys !== '') {
|
||||
$query->keys($keys);
|
||||
}
|
||||
// }
|
||||
|
||||
// Restrict the search to specific languages.
|
||||
$this->and_query->setLanguages([$lang]);
|
||||
|
||||
// Do paging.
|
||||
// $this->and_query->range($this->offset, $this->limit);
|
||||
// retrieve all results
|
||||
// $this->and_query->range(0, -1);
|
||||
|
||||
// Add sorting.
|
||||
$this->and_query->sort('search_api_relevance', 'DESC');
|
||||
$query->setLanguages([$lang]);
|
||||
$query->sort('search_api_relevance', 'DESC');
|
||||
// Tiebreaker: among equally relevant matches, show the most recently
|
||||
// modified first (Solr otherwise falls back to internal doc order, i.e.
|
||||
// indexation order, which puts the oldest materiaux first).
|
||||
$this->and_query->sort('changed', 'DESC');
|
||||
|
||||
// Set one or more tags for the query.
|
||||
$query->sort('changed', 'DESC');
|
||||
// @see hook_search_api_query_TAG_alter()
|
||||
// @see hook_search_api_results_TAG_alter()
|
||||
$this->and_query->addTag('materio_sapi_search_and_query');
|
||||
$query->addTag($tag);
|
||||
return $query;
|
||||
}
|
||||
|
||||
$and_results = $this->and_query->execute();
|
||||
|
||||
foreach ($and_results as $result) {
|
||||
// !! have to remove duplicates from phrase query
|
||||
/**
|
||||
* Executes a query and lists its nodes once each, in result order.
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid.
|
||||
*/
|
||||
private function collect($query): array {
|
||||
$list = [];
|
||||
foreach ($query->execute() as $result) {
|
||||
$nid = $result->getField('nid')->getValues()[0];
|
||||
if ( !in_array($nid, $this->results['nids']) ) {
|
||||
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
|
||||
$this->results['nids'][] = $result->getField('nid')->getValues()[0];
|
||||
}
|
||||
$list[$nid] ??= [$nid, $result->getField('uuid')->getValues()[0]];
|
||||
}
|
||||
|
||||
|
||||
$this->exactematch_count = count($this->results['nids']);
|
||||
|
||||
|
||||
|
||||
// ,---. ,---.
|
||||
// | |,---. | |. .,---.,---., .
|
||||
// | || | || ||---'| | |
|
||||
// `---'` `---\`---'`---'` `---|
|
||||
// `---'
|
||||
$this->or_query = $this->index->query(['offset'=>0,'limit'=>10000]);
|
||||
|
||||
// Change the parse mode for the search.
|
||||
// Les différentes possibilités sont
|
||||
// - « direct » => Requête directe
|
||||
// - « terms » => Multiple words
|
||||
// - « phrase » => Single phrase
|
||||
// - " edismax " => ???
|
||||
$or_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('direct');
|
||||
$or_parse_mode->setConjunction('OR');
|
||||
$this->or_query->setParseMode($or_parse_mode);
|
||||
|
||||
// Set fulltext search keywords and fields.
|
||||
if ($this->keys) {
|
||||
$this->or_query->keys(implode(' ', $this->keys));
|
||||
}
|
||||
|
||||
// // exclude results from previous and_query
|
||||
// !! trigering solr "too many boolean clauses" error
|
||||
// $exclude_and_results_conditions = $this->or_query->createConditionGroup('AND');
|
||||
// foreach ($this->results['nids'] as $nid) {
|
||||
// $exclude_and_results_conditions->addCondition('nid', $nid, '<>');
|
||||
// }
|
||||
// $this->or_query->addConditionGroup($exclude_and_results_conditions);
|
||||
|
||||
if (preg_match_all('/[WTRPCMFGSO]\d{4}/i', implode(' ', $this->keys), $matches)) {
|
||||
// in case we search for material references like W0117
|
||||
$ref_conditions = $this->or_query->createConditionGroup('OR');
|
||||
foreach ($matches[0] as $key => $value) {
|
||||
$ref_conditions->addCondition('field_reference', $value);
|
||||
}
|
||||
$this->or_query->addConditionGroup($ref_conditions);
|
||||
}
|
||||
|
||||
if ($this->filters) {
|
||||
// FILTERS
|
||||
$or_filters_conditions = $this->or_query->createConditionGroup('OR');
|
||||
foreach ($this->filters as $filter) {
|
||||
$filter = (int) $filter;
|
||||
foreach (['thesaurus_tid'] as $field) { // 'tag_tid',
|
||||
$or_filters_conditions->addCondition($field, $filter);
|
||||
}
|
||||
}
|
||||
$this->or_query->addConditionGroup($or_filters_conditions);
|
||||
|
||||
if(!$this->keys) {
|
||||
// if no keys but filters switch query to direct and add wildcard solr keys *:*
|
||||
$direct_or_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('direct');
|
||||
$direct_or_parse_mode->setConjunction('OR');
|
||||
$this->or_query->setParseMode($direct_or_parse_mode);
|
||||
// $this->or_query->keys('*:*');
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Restrict the search to specific languages.
|
||||
$this->or_query->setLanguages([$lang]);
|
||||
|
||||
// Do paging.
|
||||
// $this->or_query->range($this->offset, $this->limit);
|
||||
// retrieve all results
|
||||
// $this->or_query->range(0, -1);
|
||||
|
||||
// Add sorting.
|
||||
$this->or_query->sort('search_api_relevance', 'DESC');
|
||||
// Tiebreaker: among equally relevant matches, show the most recently
|
||||
// modified first (Solr otherwise falls back to internal doc order, i.e.
|
||||
// indexation order, which puts the oldest materiaux first).
|
||||
$this->or_query->sort('changed', 'DESC');
|
||||
|
||||
// Set one or more tags for the query.
|
||||
// @see hook_search_api_query_TAG_alter()
|
||||
// @see hook_search_api_results_TAG_alter()
|
||||
$this->or_query->addTag('materio_sapi_search_or_query');
|
||||
|
||||
$or_results = $this->or_query->execute();
|
||||
|
||||
foreach ($or_results as $result) {
|
||||
$nid = $result->getField('nid')->getValues()[0];
|
||||
// !! have to remove duplicates instead of $exclude_and_results_conditions (solr too many boolean clauses)
|
||||
if ( !in_array($nid, $this->results['nids']) ) {
|
||||
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
|
||||
$this->results['nids'][] = $nid;
|
||||
}
|
||||
}
|
||||
|
||||
// todo you may like / more like this
|
||||
|
||||
return $list;
|
||||
}
|
||||
|
||||
private function defaultQuery(){
|
||||
@@ -379,22 +373,17 @@ class Base extends ControllerBase {
|
||||
* get params from request
|
||||
*/
|
||||
private function parseRequest(Request $request){
|
||||
// Get the typed string from the URL, if it exists.
|
||||
$this->keys = $request->query->get('keys');
|
||||
if($this->keys){
|
||||
$this->keys = mb_strtolower($this->keys);
|
||||
$this->keys = Tags::explode($this->keys);
|
||||
// \Drupal::logger('materio_sapi')->notice($this->keys);
|
||||
}
|
||||
// One phrase, commas included: no more splitting into tags (H1).
|
||||
// keys stays an array of one lowercased element for the keyword queries.
|
||||
$this->phrase = self::cleanPhrase($request->query->get('keys'));
|
||||
$this->keys = $this->phrase !== '' ? [mb_strtolower($this->phrase)] : NULL;
|
||||
// get the exacte term id in case of autocomplete
|
||||
// $this->terms = $request->query->get('terms');
|
||||
$t = $request->query->get('terms');
|
||||
// $this->terms = strlen($t) ? explode(',', $t) : null;
|
||||
$this->terms = $t && strlen($t) ? json_decode($t) : null;
|
||||
|
||||
// get the filters of advanced search
|
||||
$f = $request->query->get('filters');
|
||||
$this->filters = $f && strlen($f) ? explode(',', $f) : null;
|
||||
// The filters parameter of the assisted search (old URLs) is ignored.
|
||||
// $this->allparams = $request->query->all();
|
||||
// $request->attributes->get('_raw_variables')->get('filters')
|
||||
//
|
||||
@@ -402,6 +391,17 @@ class Base extends ControllerBase {
|
||||
$this->limit = $request->query->get('limit') ?? $this->limit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes the typed phrase: spaces collapsed, length limited.
|
||||
*/
|
||||
public static function cleanPhrase($keys): string {
|
||||
if (!is_string($keys)) {
|
||||
return '';
|
||||
}
|
||||
$phrase = trim(preg_replace('/[\s\p{Cc}]+/u', ' ', $keys) ?? '');
|
||||
return mb_substr($phrase, 0, self::MAX_PHRASE_LENGTH);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handler for ajax search.
|
||||
*/
|
||||
@@ -415,14 +415,13 @@ class Base extends ControllerBase {
|
||||
),
|
||||
];
|
||||
|
||||
if ($this->keys || $this->filters) {
|
||||
if ($this->keys) {
|
||||
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
|
||||
|
||||
$this->sapiQuery();
|
||||
|
||||
$resp['keys'] = json_encode($this->keys);
|
||||
$resp['terms'] = json_encode($this->terms);
|
||||
$resp['filters'] = $this->filters;
|
||||
// $resp['count'] = $this->results->getResultCount();
|
||||
$resp['count'] = count($this->results['nids']);
|
||||
$resp['exactematch_count'] = $this->exactematch_count;
|
||||
@@ -430,28 +429,9 @@ class Base extends ControllerBase {
|
||||
"@exactmatchcount" => $resp['exactematch_count'],
|
||||
"@count" => $resp['count']
|
||||
));
|
||||
if ($this->keys) {
|
||||
$resp['infos'] .= t(' keywords @keys', array(
|
||||
"@keys" => implode(', ', $this->keys)
|
||||
));
|
||||
}
|
||||
if ($this->keys && $this->filters) {
|
||||
$resp['infos'] .= " and";
|
||||
}
|
||||
if ($this->filters) {
|
||||
// get the terms names from tid
|
||||
$storage = \Drupal::entityTypeManager()->getStorage('taxonomy_term');
|
||||
$filters_names = [];
|
||||
foreach ($this->filters as $tid) {
|
||||
/** @var \Drupal\Core\Entity\EntityInterface $entity */
|
||||
$entity = $storage->load($tid);
|
||||
$entity_trans = \Drupal::service('entity.repository')->getTranslationFromContext($entity, $lang);
|
||||
$filters_names[] = $entity_trans->getName();
|
||||
}
|
||||
$resp['infos'] .= t(' filters @filters', array(
|
||||
"@filters" => implode(', ', $filters_names)
|
||||
));
|
||||
}
|
||||
$resp['infos'] .= t(' keywords @keys', array(
|
||||
"@keys" => $this->phrase
|
||||
));
|
||||
// $resp['options'] = $this->query->getOptions();
|
||||
|
||||
// $uuids = [];
|
||||
@@ -508,7 +488,7 @@ class Base extends ControllerBase {
|
||||
];
|
||||
|
||||
if ($this->keys) {
|
||||
$resp['#title'] = implode(', ', $this->keys);
|
||||
$resp['#title'] = $this->phrase;
|
||||
|
||||
// $this->sapiQuery();
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_sapi;
|
||||
|
||||
/**
|
||||
* Keyword text of a phrase: stop words and request formulas removed (H2).
|
||||
*
|
||||
* The stop words are those of the Solr field type of the language
|
||||
* (search_api_solr text_LANG_*, the source of stopwords_LANG.txt in the
|
||||
* configset): the keyword queries also search the custom types of
|
||||
* materio_sapi (tags, thesaurus, countries, n-grams), which keep them. The
|
||||
* whole phrase still goes to the model.
|
||||
*/
|
||||
final class KeywordText {
|
||||
|
||||
/**
|
||||
* Words that only ask, and say nothing of the material.
|
||||
*/
|
||||
private const REQUEST_WORDS = [
|
||||
'fr' => ['cherche', 'recherche', 'cherchons', 'recherchons', 'besoin', 'possible', 'svp', 'merci'],
|
||||
// The English list of search_api_solr is short (no « i », « am », « my »…).
|
||||
'en' => ['i', 'am', "i'm", 'we', 'my', 'me', 'looking', 'look', 'searching', 'search', 'need', 'want', 'possible', 'something', 'please', 'thanks'],
|
||||
];
|
||||
|
||||
/**
|
||||
* Leading elisions of French, removed before the stop word check.
|
||||
*/
|
||||
private const ELISION = '/^(?:[cdjlmnst]|qu)[\'’]/u';
|
||||
|
||||
/**
|
||||
* Stop words by language, loaded once.
|
||||
*
|
||||
* @var array<string, array<string, true>>
|
||||
*/
|
||||
private static array $stopWords = [];
|
||||
|
||||
/**
|
||||
* Returns the words of a phrase that carry meaning, in order.
|
||||
*
|
||||
* @return string[]
|
||||
* Lowercased words; empty when the phrase has only stop words.
|
||||
*/
|
||||
public static function words(string $phrase, string $langcode): array {
|
||||
$skip = self::stopWords($langcode);
|
||||
$words = [];
|
||||
// Solr and Search API syntax between words is a separator (« bois:verre »,
|
||||
// « bois/métal »); hyphens, apostrophes, and . , in numbers stay.
|
||||
$phrase = preg_replace('#[:;/\\\\^&|()\[\]{}<>"«»~*?!=+\#@$]+#u', ' ', $phrase) ?? $phrase;
|
||||
foreach (preg_split('/\s+/u', mb_strtolower($phrase), -1, PREG_SPLIT_NO_EMPTY) as $token) {
|
||||
// Edge punctuation, and leading + - ! that Search API reads as operators
|
||||
// (« -bois » excluded wood and returned every other node, H3).
|
||||
$token = trim($token, ",;:!?.()[]{}\"«»'’<>");
|
||||
$token = ltrim($token, '+-!');
|
||||
$word = preg_replace(self::ELISION, '', $token) ?? $token;
|
||||
// No letter or digit (« || », « & »): breaks the payload boost of Solr.
|
||||
if (!preg_match('/[\p{L}\p{N}]/u', $word) || isset($skip[$word]) || isset($skip[$token])) {
|
||||
continue;
|
||||
}
|
||||
$words[] = $word;
|
||||
}
|
||||
return $words;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop words and request words of a language.
|
||||
*
|
||||
* @return array<string, true>
|
||||
* Words as keys.
|
||||
*/
|
||||
private static function stopWords(string $langcode): array {
|
||||
if (!isset(self::$stopWords[$langcode])) {
|
||||
$list = self::REQUEST_WORDS[$langcode] ?? [];
|
||||
$storage = \Drupal::entityTypeManager()->getStorage('solr_field_type');
|
||||
foreach (["text_{$langcode}_7_0_0", "text_{$langcode}_6_0_0"] as $id) {
|
||||
/** @var \Drupal\search_api_solr\SolrFieldTypeInterface|null $type */
|
||||
if ($type = $storage->load($id)) {
|
||||
$text = $type->getTextFiles()['stopwords'] ?? '';
|
||||
foreach (preg_split('/\R/u', $text) as $line) {
|
||||
$line = trim(preg_replace('/#.*/u', '', $line) ?? '');
|
||||
if ($line !== '') {
|
||||
$list[] = mb_strtolower($line);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
self::$stopWords[$langcode] = array_fill_keys($list, TRUE);
|
||||
}
|
||||
return self::$stopWords[$langcode];
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_sapi;
|
||||
|
||||
/**
|
||||
* Merges ranked result lists: exact matches first, the rest by RRF (H5).
|
||||
*
|
||||
* Reciprocal rank fusion: a node scores the sum of 1 / (k + rank) over the
|
||||
* lists that hold it (rank from 1). Only ranks count, not the scores of each
|
||||
* engine, which are not comparable (BM25 against cosine). Equal scores keep
|
||||
* the order of first appearance, list by list: the keyword lists are already
|
||||
* sorted by relevance then by date (fallback sort kept).
|
||||
*/
|
||||
final class RankFusion {
|
||||
|
||||
/**
|
||||
* Usual constant of RRF: dampens the weight of the first ranks.
|
||||
*/
|
||||
public const K = 60;
|
||||
|
||||
/**
|
||||
* Returns the merged order.
|
||||
*
|
||||
* @param array<int|string> $exact
|
||||
* Node IDs of the exact matches (references, phrase, all words), in order.
|
||||
* Always first, whatever the other lists say.
|
||||
* @param array<array<int|string>> $rankings
|
||||
* Ranked lists of node IDs (keyword OR query, kNN neighbours…).
|
||||
* @param int $k
|
||||
* RRF constant.
|
||||
*
|
||||
* @return array<int|string>
|
||||
* Node IDs, each once.
|
||||
*/
|
||||
public static function fuse(array $exact, array $rankings, int $k = self::K): array {
|
||||
$merged = [];
|
||||
foreach ($exact as $nid) {
|
||||
$merged[$nid] ??= $nid;
|
||||
}
|
||||
$scores = [];
|
||||
$first = [];
|
||||
$position = 0;
|
||||
foreach ($rankings as $ranking) {
|
||||
$rank = 0;
|
||||
foreach ($ranking as $nid) {
|
||||
$rank++;
|
||||
if (isset($merged[$nid])) {
|
||||
continue;
|
||||
}
|
||||
if (!isset($scores[$nid])) {
|
||||
$scores[$nid] = 0.0;
|
||||
$first[$nid] = [$position++, $nid];
|
||||
}
|
||||
$scores[$nid] += 1 / ($k + $rank);
|
||||
}
|
||||
}
|
||||
uksort($scores, static fn ($a, $b): int => [$scores[$b], $first[$a][0]] <=> [$scores[$a], $first[$b][0]]);
|
||||
foreach (array_keys($scores) as $nid) {
|
||||
$merged[$nid] = $first[$nid][1];
|
||||
}
|
||||
return array_values($merged);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\Tests\materio_sapi\Unit;
|
||||
|
||||
use Drupal\materio_sapi\RankFusion;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
// No Drupal bootstrap needed: RankFusion is plain PHP.
|
||||
require_once __DIR__ . '/../../../src/RankFusion.php';
|
||||
|
||||
/**
|
||||
* Tests the merge of the keyword and kNN rankings (H5).
|
||||
*
|
||||
* @coversDefaultClass \Drupal\materio_sapi\RankFusion
|
||||
* @group materio_sapi
|
||||
*/
|
||||
final class RankFusionTest extends TestCase {
|
||||
|
||||
/**
|
||||
* Exact matches stay first, in their order, even when ranked low elsewhere.
|
||||
*/
|
||||
public function testExactFirst(): void {
|
||||
$order = RankFusion::fuse([30, 10], [[1, 2, 10], [2, 1, 30]]);
|
||||
$this->assertSame([30, 10], array_slice($order, 0, 2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Each node appears once, exact matches included.
|
||||
*/
|
||||
public function testNoDuplicate(): void {
|
||||
$order = RankFusion::fuse([5], [[5, 1, 2], [2, 5, 3]]);
|
||||
$this->assertSame([5, 2, 1, 3], $order);
|
||||
}
|
||||
|
||||
/**
|
||||
* A node found by both lists beats a node first in one list only.
|
||||
*/
|
||||
public function testBothListsWin(): void {
|
||||
// 2 : 1/62 + 1/61 ; 1 : 1/61 ; 9 : 1/61.
|
||||
$this->assertSame([2, 1, 9], RankFusion::fuse([], [[1, 2], [9, 2]]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Equal scores keep the order of first appearance (keyword list first).
|
||||
*/
|
||||
public function testTiesKeepFirstAppearance(): void {
|
||||
$this->assertSame([7, 8, 3, 4], RankFusion::fuse([], [[7, 3], [8, 4]]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Empty lists: the exact matches alone, or nothing.
|
||||
*/
|
||||
public function testEmpty(): void {
|
||||
$this->assertSame([4], RankFusion::fuse([4], [[], []]));
|
||||
$this->assertSame([], RankFusion::fuse([], []));
|
||||
}
|
||||
|
||||
/**
|
||||
* Numeric strings and integers are the same node, the first type is kept.
|
||||
*/
|
||||
public function testIdTypes(): void {
|
||||
$this->assertSame([12, 13], RankFusion::fuse([], [[12, 13], ['12']]));
|
||||
$this->assertSame(['8'], RankFusion::fuse(['8'], [[8]]));
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user