migration is cleaning textes, added pdf migration

2026-08-13 12:21:53 +02:00
parent 914f5fd9d1
commit c19b5899e4
10 changed files with 386 additions and 3 deletions
@@ -98,6 +98,17 @@ process:
      image_field_caption/format:
        plugin: default_value
        default_value: wysiwyg
+  field_fichiers:
+    plugin: sub_process
+    source: pdfs
+    process:
+      target_id:
+        plugin: file_import
+        source: uri
+        destination: destination
+        reuse: true
+        skip_on_missing_source: true
+      description: description
  field_dates/value:
    plugin: leshed_date
    source: date_start
@@ -0,0 +1,153 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\migrate_leshed\Plugin\migrate\process;
+
+use Drupal\Core\Entity\EntityTypeManagerInterface;
+use Drupal\Core\File\FileExists;
+use Drupal\Core\File\FileSystemInterface;
+use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\migrate\Attribute\MigrateProcess;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\MigrateSkipRowException;
+use Drupal\migrate\ProcessPluginBase;
+use Drupal\migrate\Row;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Copies a local file into the file system as a managed File entity.
+ *
+ * The source value is a readable file path/URI. The file is copied into the
+ * destination directory and registered as a managed file; the file id is
+ * returned.
+ *
+ * Configuration:
+ * - destination: (required) destination directory URI, e.g. 'public://2026-08/'.
+ * - uid: owner uid of the created file (default 0).
+ * - reuse: reuse an existing managed file at the destination URI (default TRUE).
+ * - skip_on_missing_source: skip the value when the source is missing
+ *   (default TRUE) instead of failing.
+ *
+ * @code
+ * field_fichiers/target_id:
+ *   plugin: file_import
+ *   source: uri
+ *   destination: destination
+ * @endcode
+ */
+#[MigrateProcess(id: 'file_import')]
+final class FileImport extends ProcessPluginBase implements ContainerFactoryPluginInterface {
+
+  /**
+   * Per-run cache of already resolved destination URIs to file ids.
+   *
+   * @var array<string, int>
+   */
+  private array $fidCache = [];
+
+  public function __construct(
+    array $configuration,
+    $plugin_id,
+    $plugin_definition,
+    private readonly FileSystemInterface $fileSystem,
+    private readonly EntityTypeManagerInterface $entityTypeManager,
+  ) {
+    $configuration += [
+      'uid' => 0,
+      'reuse' => TRUE,
+      'skip_on_missing_source' => TRUE,
+    ];
+    parent::__construct($configuration, $plugin_id, $plugin_definition);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition): self {
+    return new self(
+      $configuration,
+      $plugin_id,
+      $plugin_definition,
+      $container->get('file_system'),
+      $container->get('entity_type.manager'),
+    );
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property): ?int {
+    $source = is_string($value) ? trim($value) : '';
+    if ($source === '') {
+      return NULL;
+    }
+
+    if (!is_file($source) && !is_file($this->fileSystem->realpath($source) ?: $source)) {
+      return $this->skipFile($migrate_executable, "Source file '$source' is missing.");
+    }
+
+    $directory = $this->getPropertyValue($this->configuration['destination'], $row);
+    if (!is_string($directory) || $directory === '') {
+      return $this->skipFile($migrate_executable, "Missing 'destination' for file '$source'.");
+    }
+    $directory = rtrim($directory, '/') . '/';
+    $destination = $directory . $this->fileSystem->basename($source);
+
+    // Reuse an already imported file to keep the migration idempotent.
+    if ($this->configuration['reuse']) {
+      if (isset($this->fidCache[$destination])) {
+        return $this->fidCache[$destination];
+      }
+      $existing = $this->entityTypeManager->getStorage('file')
+        ->loadByProperties(['uri' => $destination]);
+      if ($existing) {
+        return $this->fidCache[$destination] = (int) reset($existing)->id();
+      }
+    }
+
+    try {
+      if (!$this->fileSystem->prepareDirectory($directory, FileSystemInterface::CREATE_DIRECTORY | FileSystemInterface::MODIFY_PERMISSIONS)) {
+        return $this->skipFile($migrate_executable, "Cannot prepare directory '$directory'.");
+      }
+      $this->fileSystem->copy($source, $destination, FileExists::Replace);
+
+      $file = $this->entityTypeManager->getStorage('file')->create([
+        'uri' => $destination,
+        'uid' => (int) $this->getPropertyValue($this->configuration['uid'], $row),
+        'status' => 1,
+      ]);
+      $file->save();
+    }
+    catch (\Throwable $e) {
+      return $this->skipFile($migrate_executable, "Import of file '$source' failed: " . $e->getMessage());
+    }
+
+    return $this->fidCache[$destination] = (int) $file->id();
+  }
+
+  /**
+   * Logs a message and skips the current file item.
+   *
+   * @throws \Drupal\migrate\MigrateSkipRowException
+   */
+  private function skipFile(MigrateExecutableInterface $migrate_executable, string $message): never {
+    $migrate_executable->saveMessage($message);
+    throw new MigrateSkipRowException($message, FALSE);
+  }
+
+  /**
+   * Resolves a configuration value, supporting destination (@) references.
+   */
+  private function getPropertyValue(mixed $key, Row $row): mixed {
+    if (is_string($key) && str_starts_with($key, '@')) {
+      return $row->get(substr($key, 1));
+    }
+    if (is_string($key) && $row->hasSourceProperty($key)) {
+      return $row->getSourceProperty($key);
+    }
+
+    return $key;
+  }
+
+}
@@ -4,7 +4,9 @@ declare(strict_types=1);

 namespace Drupal\migrate_leshed\Plugin\migrate\source;

+use Drupal\Core\Entity\EntityFieldManagerInterface;
 use Drupal\Core\File\FileSystemInterface;
+use Drupal\Core\Utility\Token;
 use Drupal\migrate\Attribute\MigrateSource;
 use Drupal\migrate\Plugin\MigrationInterface;
 use Drupal\migrate\Row;
@@ -46,6 +48,11 @@ final class ProjetNode extends Url {
   */
  private const IMAGE_EXTENSIONS = ['png', 'gif', 'jpg', 'jpeg', 'webp'];

+  /**
+   * File extensions to import into field_fichiers.
+   */
+  private const FILE_EXTENSIONS = ['pdf'];
+
  /**
   * Default destination directory when no constant is configured.
   */
@@ -63,6 +70,11 @@ final class ProjetNode extends Url {
   */
  private ?array $folderMap = NULL;

+  /**
+   * Memoized destination directory for field_fichiers (from field settings).
+   */
+  private ?string $filesDestination = NULL;
+
  public function __construct(
    array $configuration,
    $plugin_id,
@@ -70,6 +82,8 @@ final class ProjetNode extends Url {
    MigrationInterface $migration,
    DataParserPluginManager $parserPluginManager,
    private readonly FileSystemInterface $fileSystem,
+    private readonly EntityFieldManagerInterface $entityFieldManager,
+    private readonly Token $token,
  ) {
    parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $parserPluginManager);
  }
@@ -85,6 +99,8 @@ final class ProjetNode extends Url {
      $migration,
      $container->get('plugin.manager.migrate_plus.data_parser'),
      $container->get('file_system'),
+      $container->get('entity_field.manager'),
+      $container->get('token'),
    );
  }

@@ -95,6 +111,7 @@ final class ProjetNode extends Url {
    return parent::fields() + [
      'body_html' => $this->t('Project body (HTML converted from Markdown)'),
      'images' => $this->t('Project images with captions'),
+      'pdfs' => $this->t('Project PDF files'),
    ];
  }

@@ -123,12 +140,14 @@ final class ProjetNode extends Url {
    // an undefined source key, even when no drive folder exists yet.
    $row->setSourceProperty('body_html', NULL);
    $row->setSourceProperty('images', []);
+    $row->setSourceProperty('pdfs', []);

    $uid = trim((string) $row->getSourceProperty('id'));
    $folder = $uid !== '' ? $this->findProjectFolder($uid) : NULL;
    if ($folder !== NULL) {
      $row->setSourceProperty('body_html', $this->readBody($folder, $uid));
      $row->setSourceProperty('images', $this->readImages($folder, $uid, (string) $row->getSourceProperty('title')));
+      $row->setSourceProperty('pdfs', $this->readPdfs($folder, $uid));
    }

    return TRUE;
@@ -188,8 +207,7 @@ final class ProjetNode extends Url {
    sort($entries, SORT_NATURAL | SORT_FLAG_CASE);
    $markdown = NULL;
    foreach ($entries as $entry) {
-      // Ignore ".back.md" backups of the pre-clean source text.
-      if (!is_file($dir . '/' . $entry) || !preg_match('/\.md$/i', $entry) || preg_match('/\.back\.md$/i', $entry)) {
+      if (!is_file($dir . '/' . $entry) || !preg_match('/\.md$/i', $entry)) {
        continue;
      }
      if (preg_match($preferred, $entry)) {
@@ -206,10 +224,44 @@ final class ProjetNode extends Url {
    if ($contents === FALSE || trim($contents) === '') {
      return NULL;
    }
+    $contents = $this->cleanBodyMarkdown($contents);
+    if ($contents === '') {
+      return NULL;
+    }

    return (string) $this->getMarkdownConverter()->convert($contents);
  }

+  /**
+   * Cleans the raw source Markdown before conversion.
+   *
+   * Removes redundant content already present elsewhere: raw HTML wrappers,
+   * invisible characters, and the leading title block (H1 title + H2/H3
+   * artists/subtitle, redundant with the sheet). Headings located *inside* the
+   * body (e.g. artist/curator bios) are preserved. Idempotent: running it on an
+   * already-cleaned text has no effect.
+   */
+  private function cleanBodyMarkdown(string $md): string {
+    // 1. Strip HTML tags (<div>, <br>, </div>, ...).
+    $md = (string) preg_replace('/<[^>]+>/', '', $md);
+    // 2. Strip zero-width/BOM characters and turn NBSP into a normal space.
+    $md = (string) preg_replace('/[\x{200B}\x{200E}\x{200F}\x{FEFF}]/u', '', $md);
+    $md = (string) preg_replace('/\x{00A0}/u', ' ', $md);
+    // 3. Drop only the leading heading block: skip leading blank and heading
+    //    lines until the first body line.
+    $lines = preg_split('/\R/u', $md) ?: [];
+    $i = 0;
+    $count = count($lines);
+    while ($i < $count && (trim($lines[$i]) === '' || preg_match('/^\s*#{1,6}\s/u', $lines[$i]))) {
+      $i++;
+    }
+    $md = implode("\n", array_slice($lines, $i));
+    // 4. Collapse blank lines and trim.
+    $md = (string) preg_replace('/\n{3,}/', "\n\n", $md);
+
+    return trim($md);
+  }
+
  /**
   * Builds the list of images (URI + alt) for the project.
   *
@@ -272,6 +324,90 @@ final class ProjetNode extends Url {
    return $images;
  }

+  /**
+   * Builds the list of PDF files for the project.
+   *
+   * Collects PDFs both at the project folder root and inside the "{UID}_pdf"
+   * subfolder. Each item carries the destination directory (from the
+   * field_fichiers settings) and a human-readable description.
+   *
+   * @return array<int, array{uri: string, destination: string, description: string}>
+   *   The files to import.
+   */
+  private function readPdfs(string $folderUri, string $uid): array {
+    $dir = $this->fileSystem->realpath($folderUri);
+    if ($dir === FALSE) {
+      return [];
+    }
+
+    // Directories to scan: the project folder root and its "{UID}_pdf" subfolder.
+    $sources = [$folderUri];
+    $pattern = '/^' . preg_quote($uid, '/') . '_pdf$/i';
+    foreach (scandir($dir) ?: [] as $entry) {
+      if (preg_match($pattern, $entry) && is_dir($dir . '/' . $entry)) {
+        $sources[] = $folderUri . '/' . $entry;
+      }
+    }
+
+    $destination = $this->filesDestination();
+    $pdfs = [];
+    foreach ($sources as $sourceUri) {
+      $sourceDir = $this->fileSystem->realpath($sourceUri);
+      if ($sourceDir === FALSE) {
+        continue;
+      }
+      $files = scandir($sourceDir) ?: [];
+      sort($files, SORT_NATURAL | SORT_FLAG_CASE);
+      foreach ($files as $file) {
+        if (!is_file($sourceDir . '/' . $file)) {
+          continue;
+        }
+        if (!in_array(strtolower(pathinfo($file, PATHINFO_EXTENSION)), self::FILE_EXTENSIONS, TRUE)) {
+          continue;
+        }
+        $pdfs[] = [
+          'uri' => $sourceUri . '/' . $file,
+          'destination' => $destination,
+          'description' => $this->humanizeFilename($file),
+        ];
+      }
+    }
+
+    return $pdfs;
+  }
+
+  /**
+   * Resolves (once) the field_fichiers destination directory from its settings.
+   */
+  private function filesDestination(): string {
+    if ($this->filesDestination !== NULL) {
+      return $this->filesDestination;
+    }
+    $scheme = 'public';
+    $directory = '';
+    $definitions = $this->entityFieldManager->getFieldDefinitions('node', 'projet');
+    $field = $definitions['field_fichiers'] ?? NULL;
+    if ($field !== NULL) {
+      $scheme = $field->getFieldStorageDefinition()->getSetting('uri_scheme') ?: 'public';
+      $directory = (string) $this->token->replace((string) $field->getSetting('file_directory'));
+    }
+
+    return $this->filesDestination = $scheme . '://' . trim($directory, '/') . '/';
+  }
+
+  /**
+   * Turns a file name into a readable label (drops UID prefix and extension).
+   */
+  private function humanizeFilename(string $file): string {
+    $name = pathinfo($file, PATHINFO_FILENAME);
+    // Drop a leading "{UID}_" numeric prefix.
+    $name = preg_replace('/^\d+[_-]/', '', $name) ?? $name;
+    // Separators to spaces, collapse, trim.
+    $name = (string) preg_replace('/[_\-]+/u', ' ', $name);
+
+    return trim((string) preg_replace('/\s+/u', ' ', $name));
+  }
+
  /**
   * Returns the raw contents of a caption (legende) file, or '' when missing.
   */