Import de relevé bancaire CSV en libre-service (/lignes/importer-releve)

Chaque transaction du relevé devient une ligne « à trier » : type autre,
répartition vide (liseré rouge existant via field_ecart), tag field_flag
« IMP AAMMJJ » (liseré ambre + filtre existants), client rapproché par
mots (ClientMatcher : séquence contiguë, sinon mot significatif unique --
jamais en sous-chaîne, jamais auto-créé).

- src/Import/ : CsvReleveParser (ISO-8859-1 confirmé, en-tête strict,
  fgetcsv avec escape '' explicite -- dépréciation PHP 8.4, rejets
  propres avec n° de ligne), ReleveImportBatch (Batch API par lots de
  25, dédoublonnage COMPTÉ par empreinte field_import_fitid -- max(0,
  k−m) importe les vrais doublons légitimes et dédoublonne à travers
  des fichiers qui se chevauchent --, totaux de contrôle au centime
  sur la page de résultat, résumé en tempstore privé).
- ReleveUploadForm : upload private://releves (fichier conservé +
  usage, hors de portée du cron), parse en validateForm(), batch,
  redirection vers la page de résultat.
- field_montant_releve : référence bancaire immuable, écrite une fois
  à l'import et jamais par presave ; affichée en TEXTE sous Montant
  TTC (widget remplacé par un #type item -- un item ne soumet rien et
  extractFormValues() saute le champ sans valeur soumise, la valeur
  survit donc à chaque save) ; masquée sur les lignes sans montant.
- SkipValidationContext : contournement du contrôle de répartition
  requête-scopé (ferme le trou de concurrence de l'ancien state
  global, AUDIT-2026-09-09 §2.2) -- presave honore le service (state
  gardé pour compat), updateType/updateField basculent dessus.
- Permissions (AUDIT §2.2 priorité 1) : access figli ledger sur toutes
  les routes du module + autocomplete (RouteSubscriber), import
  réservé Éditeur/Admin, access content retiré du rôle Authenticated
  (config/sync re-exportée pour les 4 rôles).
- Gin : hook_gin_ignore_sticky_form_actions() -- sans ça, le bouton
  Importer partait dans la barre sticky du chrome masqué.
- install : 8011 champs, 8012 index sur les empreintes, 8013/8014
  montant_releve sur le formulaire sous le TTC (poids renumérotés).
- /lignes : boutons + Ajouter / Importer / Historique dans le footer
  sticky (compacts), footer colspan dès la première colonne, badges de
  signalement qui reviennent à la ligne au lieu de déborder.
This commit is contained in:
2026-09-09 14:58:36 +02:00
parent 3688bddba8
commit d2f3179f01
27 changed files with 1434 additions and 57 deletions
@@ -18,6 +18,7 @@ class DashboardController extends ControllerBase {
return [
'#theme' => 'figli_compta_home',
'#can_view_history' => $this->currentUser()->hasPermission('view ligne_comptable revisions'),
'#can_import_releve' => $this->currentUser()->hasPermission('import ligne_comptable releve'),
'#current_route' => 'figli_compta_ledger.home',
'#attached' => [
'library' => ['figli_compta_ledger/home'],
@@ -5,8 +5,10 @@ namespace Drupal\figli_compta_ledger\Controller;
use Drupal\Core\Access\CsrfRequestHeaderAccessCheck;
use Drupal\Core\Controller\ControllerBase;
use Drupal\Core\Entity\EntityStorageException;
use Drupal\figli_compta_ledger\SkipValidationContext;
use Drupal\node\NodeInterface;
use Drupal\taxonomy\Entity\Term;
use Symfony\Component\DependencyInjection\ContainerInterface;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
@@ -23,6 +25,24 @@ use Symfony\Component\HttpFoundation\Request;
*/
class LedgerActionsController extends ControllerBase {
/**
* Request-scoped répartition-check opt-out -- see the class docblock of
* \Drupal\figli_compta_ledger\SkipValidationContext for why this replaced
* the historical global state key here.
*
* @var \Drupal\figli_compta_ledger\SkipValidationContext
*/
protected $skipValidationContext;
/**
* {@inheritdoc}
*/
public static function create(ContainerInterface $container) {
$instance = parent::create($container);
$instance->skipValidationContext = $container->get('figli_compta_ledger.skip_validation_context');
return $instance;
}
/**
* Fields editable inline from /lignes without opening the full node
* edit form -- keys are the short names the frontend sends; values are
@@ -96,20 +116,17 @@ class LedgerActionsController extends ControllerBase {
// (from historical data, never corrected -- see figli_compta_ledger's
// module docblock) exactly as it was. The présave check exists to
// catch new inconsistent entries, not to block relabeling the type of
// an already-migrated line, so skip it for this save only. try/finally
// guarantees the global flag clears even if save() throws for an
// unrelated reason -- leaving it on would silently skip validation on
// every other save on the site.
\Drupal::state()->set('figli_compta_ledger.skip_validation', TRUE);
// an already-migrated line, so skip it for this save only. The
// SkipValidationContext service is request-scoped with a try/finally
// inside skip(), so the check is back on the instant save() returns
// or throws -- no global flag left hanging that a concurrent save
// from someone else could fall into.
try {
$node->save();
$this->skipValidationContext->skip(fn () => $node->save());
}
catch (EntityStorageException $e) {
return new JsonResponse(['error' => $e->getMessage()], 422);
}
finally {
\Drupal::state()->delete('figli_compta_ledger.skip_validation');
}
return new JsonResponse([
'success' => TRUE,
@@ -167,17 +184,14 @@ class LedgerActionsController extends ControllerBase {
// signalement changes here, montant_ht and field_repartition are
// untouched, so skipping the répartition check for this save can
// never introduce a mismatch -- it can only leave a pre-existing
// historical one exactly as it was.
\Drupal::state()->set('figli_compta_ledger.skip_validation', TRUE);
// historical one exactly as it was. Request-scoped skip (see
// updateType()'s comment), no global flag.
try {
$node->save();
$this->skipValidationContext->skip(fn () => $node->save());
}
catch (EntityStorageException $e) {
return new JsonResponse(['error' => $e->getMessage()], 422);
}
finally {
\Drupal::state()->delete('figli_compta_ledger.skip_validation');
}
if ($field === 'client') {
$newValue = $node->get('field_client')->entity ? $node->get('field_client')->entity->label() : NULL;
@@ -0,0 +1,72 @@
<?php
namespace Drupal\figli_compta_ledger\Controller;
use Drupal\Core\Controller\ControllerBase;
use Drupal\Core\Routing\LocalRedirectResponse;
use Drupal\Core\Url;
/**
* Post-import summary page: counts, the accounting control total
* (file total == created + duplicates, to the centime), the list of
* duplicates skipped for human review, and a deep link into /lignes
* filtered on this batch's "IMP AAMMJJ" tag.
*
* The numbers live one-shot in the private tempstore (written by
* ReleveImportBatch::finished(), read + purged here): the page is only
* meaningful right after an import, and every user sees their own.
*/
class ReleveImportResultController extends ControllerBase {
/**
* Renders the summary of the import that just ran.
*/
public function result() {
$store = \Drupal::service('tempstore.private')->get('figli_compta_ledger');
$summary = $store->get('releve_import_result');
if (!$summary) {
// Direct navigation (bookmark, back button long after the import):
// no numbers in memory anymore, send back to the form instead of
// showing an empty shell.
$this->messenger()->addWarning($this->t("Le résultat d'un import n'est disponible qu'immédiatement après l'import lui-même."));
return new LocalRedirectResponse(Url::fromRoute('figli_compta_ledger.releve_import_form')->toString());
}
$store->delete('releve_import_result');
$eur = fn ($x) => number_format((float) $x, 2, ',', ' ') . ' €';
$fr_date = fn ($iso) => preg_replace('/^(\d{4})-(\d{2})-(\d{2})$/', '$3/$2/$1', (string) $iso);
$view = [
'file_name' => $summary['file_name'],
'tag' => $summary['tag'],
'created' => (int) $summary['created'],
'duplicates' => (int) $summary['duplicates'],
'matched' => (int) $summary['matched'],
'unmatched' => (int) $summary['unmatched'],
'errors_count' => count($summary['errors']),
'errors' => array_map(fn ($e) => $e['libelle'] . ' — ' . $e['error'], $summary['errors']),
'duplicates_list' => array_map(fn ($d) => [
'date' => $fr_date($d['date']),
'montant' => $eur($d['montant']),
'libelle' => $d['libelle'],
], $summary['duplicates_list']),
'file_total' => $eur($summary['file_total']),
'created_total' => $eur($summary['created_total']),
'duplicates_total' => $eur($summary['duplicates_total']),
'totals_ok' => (bool) $summary['totals_ok'],
];
return [
'#theme' => 'figli_compta_releve_import_result',
'#summary' => $view,
// /lignes reads its filter state from location.hash -- the flag
// filter key is "tag" (see readHashState() in js/home.js), values
// are comma-separated flag names.
'#lignes_url' => Url::fromRoute('figli_compta_ledger.home')->toString() . '#tag=' . rawurlencode($summary['tag']),
'#import_url' => Url::fromRoute('figli_compta_ledger.releve_import_form')->toString(),
'#attached' => ['library' => ['figli_compta_ledger/releve_import']],
'#cache' => ['max-age' => 0],
];
}
}
@@ -24,7 +24,11 @@ class RouteSubscriber extends RouteSubscriberBase {
*/
protected function alterRoutes(RouteCollection $collection) {
if ($route = $collection->get('system.entity_autocomplete')) {
$route->setRequirement('_permission', 'access content');
// 'access figli ledger' rather than 'access content': the generic
// Authenticated role no longer holds the latter (removed 2026-09),
// and anyone entitled to see ledger autocomplete suggestions must
// be entitled to the ledger's data itself.
$route->setRequirement('_permission', 'access figli ledger');
}
}
@@ -0,0 +1,184 @@
<?php
namespace Drupal\figli_compta_ledger\Form;
use Drupal\Core\Form\FormBase;
use Drupal\Core\Form\FormStateInterface;
use Drupal\file\Entity\File;
use Drupal\figli_compta_ledger\Import\CsvReleveParser;
use Drupal\figli_compta_ledger\Import\ReleveImportBatch;
use Drupal\figli_compta_ledger\Import\ReleveTransaction;
use Drupal\taxonomy\Entity\Term;
/**
* Upload form for a bank statement export (CSV v1 -- see
* PLAN-import-releve-bancaire.md). Self-service for the associates: the
* file lands in the *private* filesystem (financial data), is parsed in
* memory, deduplicated count-aware against lines already in base, then
* turned into "à trier" draft lines by ReleveImportBatch. Nothing here
* writes ledger lines directly -- everything goes through the normal
* Node::save() lifecycle under SkipValidationContext.
*/
class ReleveUploadForm extends FormBase {
/**
* {@inheritdoc}
*/
public function getFormId(): string {
return 'figli_compta_ledger_releve_upload_form';
}
/**
* {@inheritdoc}
*/
public function buildForm(array $form, FormStateInterface $form_state): array {
$form['releve_file'] = [
'#type' => 'managed_file',
'#title' => $this->t('Relevé bancaire (CSV)'),
'#upload_location' => 'private://releves',
// Only .csv: the OFX/CMI exports of the same account exist but
// are explicitly out of v1 scope (truncated labels / unstable
// structure -- see the plan). A clear message beats a silent
// failure for someone uploading them by mistake.
'#upload_validators' => [
'FileExtension' => ['extensions' => 'csv'],
],
'#required' => TRUE,
'#description' => $this->t('Export CSV de la banque : colonnes « Date ; Date de valeur ; Débit ; Crédit ; Libellé ; Solde » (les fichiers .ofx et .cmi ne sont pas pris en charge pour l\'instant). Chaque transaction devient une ligne « à trier » : type, répartition et HT/TVA restent à assigner à la main.'),
];
$form['actions'] = ['#type' => 'actions'];
$form['actions']['submit'] = [
'#type' => 'submit',
'#value' => $this->t('Importer le relevé'),
'#button_type' => 'primary',
];
return $form;
}
/**
* {@inheritdoc}
*
* The idiomatic home for the parse: an invalid file is a validation
* error, rejected before anything is written (the file entity is only
* promoted in submitForm()). The parsed transactions are stashed in
* $form_state so the file is never parsed twice.
*/
public function validateForm(array &$form, FormStateInterface $form_state): void {
$fids = $form_state->getValue('releve_file');
$fids = is_array($fids) ? $fids : [];
if (!$fids) {
// #required already covers the empty case.
return;
}
$file = File::load(reset($fids));
if (!$file) {
$form_state->setErrorByName('releve_file', $this->t("Le fichier téléversé n'a pas pu être retrouvé."));
return;
}
// Parse (pure, no writes). A clean form error -- never a crash page
// -- for anything the parser rejects.
try {
$transactions = (new CsvReleveParser())->parse($file->getFileUri());
}
catch (\RuntimeException $e) {
$form_state->setErrorByName('releve_file', $e->getMessage());
return;
}
$form_state->set('releve_fid', (int) $file->id());
$form_state->set('releve_transactions', array_map(fn ($t) => $t->toArray(), $transactions));
}
/**
* {@inheritdoc}
*/
public function submitForm(array &$form, FormStateInterface $form_state): void {
$file = File::load($form_state->get('releve_fid'));
$transactions = array_map([ReleveTransaction::class, 'fromArray'], $form_state->get('releve_transactions') ?: []);
if (!$file || !$transactions) {
// Normally unreachable -- validateForm() blocks bad files before
// submit is reached. Defensive only.
$form_state->setErrorByName('releve_file', $this->t("Rien à importer : relancez l'upload."));
return;
}
// Keep the uploaded statement permanently + registered as our usage:
// it's accounting source material, cron must not garbage-collect it
// after a few hours as it would a temporary file.
$file->setPermanent();
$file->save();
\Drupal::service('file.usage')->add($file, 'figli_compta_ledger', 'releve_import', (int) $file->id());
// Count-aware dedup: how many times each fingerprint appears in this
// file (k), one grouped query for how many already exist in base
// (m, all provenances combined), quota = max(0, k m).
$counts = [];
foreach ($transactions as $t) {
$counts[$t->fitid] = ($counts[$t->fitid] ?? 0) + 1;
}
$db_counts = [];
if ($counts) {
$select = \Drupal::database()->select('node__field_import_fitid', 'f')
->condition('f.field_import_fitid_value', array_keys($counts), 'IN');
$select->addField('f', 'field_import_fitid_value', 'fitid');
$select->addExpression('COUNT(*)', 'n');
$select->groupBy('f.field_import_fitid_value');
foreach ($select->execute()->fetchAllKeyed() as $fitid => $n) {
$db_counts[$fitid] = (int) $n;
}
}
$quotas = [];
foreach ($counts as $fitid => $k) {
$quotas[$fitid] = max(0, $k - ($db_counts[$fitid] ?? 0));
}
// One "IMP AAMMJJ" flag term per import batch (day granularity: the
// same day's re-imports join the same lot) -- short on purpose, it
// renders as a badge in /lignes' narrow Signalement column. The
// associates sort lines through the existing signalement mechanism
// (filter + amber marker), zero new UI.
$tag = 'IMP ' . date('ymd');
$terms = \Drupal::entityTypeManager()->getStorage('taxonomy_term')
->loadByProperties(['vid' => 'flag', 'name' => $tag]);
if ($terms) {
$term = reset($terms);
}
else {
$term = Term::create(['vid' => 'flag', 'name' => $tag]);
$term->save();
}
$file_total = 0.0;
$payload = [];
foreach ($transactions as $t) {
$file_total += $t->montant;
$payload[] = $t->toArray();
}
batch_set([
'title' => $this->t('Import du relevé bancaire'),
'operations' => [
[
[ReleveImportBatch::class, 'operation'],
[$payload, $quotas, [
'file_name' => $file->getFilename(),
'file_total' => round($file_total, 2),
'tag' => $tag,
'flag_tid' => (int) $term->id(),
]],
],
],
'finished' => [ReleveImportBatch::class, 'finished'],
'init_message' => $this->t('Import du relevé en cours…'),
'progress_message' => $this->t('@current/@total'),
'error_message' => $this->t('L\'import a rencontré une erreur inattendue.'),
]);
// Where the browser lands once the batch is done -- the result page
// reads its numbers from the private tempstore.
$form_state->setRedirect('figli_compta_ledger.releve_import_result');
}
}
@@ -0,0 +1,144 @@
<?php
namespace Drupal\figli_compta_ledger\Import;
use Drupal\taxonomy\Entity\Term;
/**
* Suggests which client term a bank label refers to -- conservatively:
* an empty match a human fills in beats a wrong match nobody re-checks
* (the same philosophy as the whole import feature: pre-fill, never
* decide).
*
* Matching is word-based, never raw substring: after normalization
* (uppercase, accents removed, non-alphanumerics as separators) a
* client's full name must appear as a contiguous word *sequence* in the
* label ("OVH SAS" matches "PRLV SEPA OVH SAS TWLN…" but a hypothetical
* client "AIR" would NOT match "CLAIR" -- the v0 substring draft of this
* plan had exactly that false-positive mode for short names).
*
* Pass 1: full normalized client name as contiguous word sequence. Two
* distinct clients matching is ambiguous empty.
* Pass 2 (only if pass 1 found nothing): a single "significant" word
* ( 4 chars, not a legal-form filler like SAS/SARL) that belongs to
* exactly ONE client in the whole vocabulary. Several candidate clients
* empty. Deliberately recall-biased: a generic-but-unique word (say
* "MAISON", held by a single client) can suggest the wrong client for
* an unrelated label -- acceptable because every imported line is
* flagged and manually sorted (see ReleveImportBatch), so a wrong
* suggestion gets corrected by a human rather than trusted.
*/
final class ClientMatcher {
/**
* Legal-form filler words never significant on their own.
*/
private const STOPWORDS = ['SAS', 'SARL', 'SA', 'SASU', 'EURL', 'SCI', 'SCOP', 'ASSOCIATION', 'GMBH', 'SNC'];
/**
* Loaded client vocabulary, shape: [['term' => Term, 'words' => string[]]].
*
* @var array|null
*/
private ?array $clients = NULL;
/**
* Returns the client term a bank label most likely refers to, or NULL
* when nothing safe can be said. Never creates a term (unlike flag
* auto-creation) -- the client vocabulary stays curated by hand.
*/
public function match(string $libelle): ?Term {
$words = $this->words($libelle);
if (!$words) {
return NULL;
}
$clients = $this->loadClients();
if (!$clients) {
return NULL;
}
// Pass 1: full name as a contiguous word sequence, unique candidate.
$pass1 = [];
foreach ($clients as $client) {
if (self::containsSequence($words, $client['words'])) {
$pass1[$client['term']->id()] = $client['term'];
}
}
if (count($pass1) === 1) {
return reset($pass1);
}
if (count($pass1) > 1) {
return NULL;
}
// Pass 2: a significant word owned by exactly one client vocabulary.
$pass2 = [];
foreach ($clients as $client) {
foreach ($client['words'] as $word) {
if (mb_strlen($word) < 4 || in_array($word, self::STOPWORDS, TRUE)) {
continue;
}
if (in_array($word, $words, TRUE)) {
$pass2[$client['term']->id()] = $client['term'];
break;
}
}
}
if (count($pass2) === 1) {
return reset($pass2);
}
return NULL;
}
/**
* Uppercase, accent-free word tokens: "EPAU / POPSU" ["EPAU","POPSU"].
*
* @return string[]
*/
private function words(string $text): array {
$transliterated = \Drupal::transliteration()->transliterate($text, 'fr');
$upper = mb_strtoupper($transliterated);
$words = preg_split('/[^A-Z0-9]+/', $upper, -1, PREG_SPLIT_NO_EMPTY);
return $words === FALSE ? [] : $words;
}
/**
* Loads (once per request) every client term with its normalized words.
*/
private function loadClients(): array {
if ($this->clients !== NULL) {
return $this->clients;
}
$this->clients = [];
$terms = \Drupal::entityTypeManager()->getStorage('taxonomy_term')
->loadByProperties(['vid' => 'client']);
foreach ($terms as $term) {
$words = $this->words($term->label());
if ($words) {
$this->clients[] = ['term' => $term, 'words' => $words];
}
}
return $this->clients;
}
/**
* Whether $needle appears in $haystack as a contiguous word sequence.
*/
private static function containsSequence(array $haystack, array $needle): bool {
$n = count($needle);
$h = count($haystack);
if ($n === 0 || $n > $h) {
return FALSE;
}
for ($i = 0; $i <= $h - $n; $i++) {
for ($j = 0; $j < $n; $j++) {
if ($haystack[$i + $j] !== $needle[$j]) {
continue 2;
}
}
return TRUE;
}
return FALSE;
}
}
@@ -0,0 +1,160 @@
<?php
namespace Drupal\figli_compta_ledger\Import;
/**
* Parses the CSV export of the SAS bank account into ReleveTransaction
* objects. Built from (and verified against) the real sample in
* sources-compta/extrais de comptes/00021322002.csv: `;`-separated,
* ISO-8859-1 encoded, columns "Date;Date de valeur;Débit;Crédit;Libellé;
* Solde", dates JJ/MM/AAAA, French decimal comma, exactly one of
* Débit/Crédit filled per row. The Solde column is ignored (no balance
* reconciliation in v1).
*
* Any deviation (wrong header, unparsable date/amount, empty label) throws
* a RuntimeException with a clear, user-facing French message -- the upload
* form catches it and shows a form error, never a raw crash page. Nothing
* is written to the database from here: parsing is a pure in-memory step.
*/
final class CsvReleveParser {
/**
* The exact header (after ISO-8859-1 UTF-8 conversion) a file must
* carry to be considered a supported statement export.
*/
private const HEADER = ['Date', 'Date de valeur', 'Débit', 'Crédit', 'Libellé', 'Solde'];
/**
* Parses a file by URI (any stream wrapper, typically private://).
*
* @return \Drupal\figli_compta_ledger\Import\ReleveTransaction[]
* Every data row as a transaction, in file order.
*
* @throws \RuntimeException
* With a ready-to-display message when the file isn't a supported
* statement export.
*/
public function parse(string $uri): array {
$stream = @fopen($uri, 'r');
if ($stream === FALSE) {
throw new \RuntimeException("Le fichier téléversé n'a pas pu être relu depuis le stockage privé.");
}
// Confirmed ISO-8859-1 on the real sample: a naive UTF-8 read would
// corrupt every accented label -- the one truly silent bug risk of
// this parser. The filter converts as fgetcsv() reads.
stream_filter_append($stream, 'convert.iconv.ISO-8859-1/UTF-8');
// Explicit enclosure + empty $escape: PHP 8.4 deprecates relying on
// the default escape (backslash), whose legacy behavior would let a
// stray "\" in a bank label swallow the next character -- with ''
// the bank's own quotes stay the only special characters.
$header = fgetcsv($stream, NULL, ';', '"', '');
if ($header === FALSE) {
fclose($stream);
throw new \RuntimeException('Le fichier est vide.');
}
$header = array_map(fn ($h) => trim((string) $h), $header);
if ($header !== self::HEADER) {
fclose($stream);
throw new \RuntimeException('Format de fichier non reconnu. En-tête attendu : « ' . implode(';', self::HEADER) . ' ». Seul l\'export CSV de la banque est pris en charge pour l\'instant (.ofx et .cmi non encore).');
}
$transactions = [];
$line = 1;
while (($row = fgetcsv($stream, NULL, ';', '"', '')) !== FALSE) {
$line++;
// Fully blank rows are just padding at the end of some exports.
if (trim(implode('', array_map('strval', $row))) === '') {
continue;
}
if (count($row) < 6) {
fclose($stream);
throw new \RuntimeException("Ligne $line : nombre de colonnes inattendu (" . count($row) . ", 6 attendues).");
}
$dateRaw = trim((string) $row[0]);
if (!preg_match('/^(\d{2})\/(\d{2})\/(\d{4})$/', $dateRaw, $m) || !checkdate((int) $m[2], (int) $m[1], (int) $m[3])) {
fclose($stream);
throw new \RuntimeException("Ligne $line : date « $dateRaw » invalide (JJ/MM/AAAA attendu).");
}
$date = $m[3] . '-' . $m[2] . '-' . $m[1];
$debit = trim((string) $row[2]);
$credit = trim((string) $row[3]);
if ($debit !== '' && $credit !== '') {
fclose($stream);
throw new \RuntimeException("Ligne $line : Débit et Crédit renseignés simultanément, format inattendu.");
}
// A Débit is money out whatever sign the bank exported it with
// (the sample already stores it negative; -abs() normalizes any
// sibling export that doesn't), a Crédit is money in.
if ($debit !== '') {
$montant = -abs($this->parseAmount($debit, $line));
}
elseif ($credit !== '') {
$montant = abs($this->parseAmount($credit, $line));
}
else {
fclose($stream);
throw new \RuntimeException("Ligne $line : ni Débit ni Crédit renseigné.");
}
$libelle = self::normalizeLibelle((string) $row[4]);
if ($libelle === '') {
fclose($stream);
throw new \RuntimeException("Ligne $line : libellé vide, impossible de tracer la transaction.");
}
$transactions[] = new ReleveTransaction(
$date,
$montant,
$libelle,
self::fitid($date, $montant, $libelle),
);
}
fclose($stream);
if (!$transactions) {
throw new \RuntimeException("Aucune transaction trouvée dans le fichier (en-tête seul).");
}
return $transactions;
}
/**
* Fingerprint of one transaction: 'csv:' + sha1(date | signed amount to
* the centime | whitespace-normalized label). No case-folding -- two
* exports of the same account reproduce labels byte for byte, and the
* fingerprint must stay stable for the count-aware dedup to recognize
* an already-imported transaction years later.
*
* The amount goes in as a fixed 2-decimal string ("1234.56") so no
* floating-point representation ever enters the hash.
*/
public static function fitid(string $date, float $montant, string $normalizedLibelle): string {
return 'csv:' . sha1($date . '|' . number_format($montant, 2, '.', '') . '|' . $normalizedLibelle);
}
/**
* Trim + collapse internal whitespace runs to one space: stray double
* spaces would otherwise make the same transaction fingerprint
* differently across two exports of the same account.
*/
public static function normalizeLibelle(string $libelle): string {
return trim((string) preg_replace('/\s+/u', ' ', $libelle));
}
/**
* French decimal ("1 234,56", "-45,89") float, with a hard format
* check -- anything unexpected rejects the whole file with the line
* number rather than being silently coerced.
*/
private function parseAmount(string $raw, int $line): float {
$clean = str_replace([' ', "\xC2\xA0"], '', $raw);
$clean = str_replace(',', '.', $clean);
if (!preg_match('/^[+-]?\d+(\.\d+)?$/', $clean)) {
throw new \RuntimeException("Ligne $line : montant « $raw » invalide.");
}
return (float) $clean;
}
}
@@ -0,0 +1,181 @@
<?php
namespace Drupal\figli_compta_ledger\Import;
use Drupal\Core\Entity\EntityStorageException;
use Drupal\node\Entity\Node;
use Drupal\taxonomy\Entity\Term;
/**
* Batch backend of the bank statement import: turns the parsed
* transactions into ligne_comptable nodes, ~25 per PHP-FPM request
* (several hundred transactions would blow the memory/time budget of a
* single request -- no Batch API precedent existed in this module
* before this).
*
* Count-aware dedup (see PLAN-import-releve-bancaire.md): for every
* fingerprint, the file tells how many times the transaction appears (k)
* and the database how many are already imported (m) -- the batch then
* creates max(0, k m) lines. That imports every legitimate duplicate
* (two identical transfers the same day) while still recognizing an
* already-imported transaction across overlapping files.
*
* Every created line goes through SkipValidationContext (request-scoped,
* NOT the historical state key): répartition is deliberately empty, so
* the presave invariant sum(répartition) == montant_ht must not fire --
* field_ecart (= montant_ht) still gets computed and drives the existing
* "à trier" red marker on /lignes.
*/
final class ReleveImportBatch {
/**
* Transactions processed per batch step.
*/
const CHUNK = 25;
/**
* Batch operation -- called repeatedly by Drupal until finished.
*
* @param array $transactions
* ReleveTransaction::toArray() payloads, in file order.
* @param array $quotas
* fitid => remaining lines to create (k m, floored at 0).
* @param array $meta
* Immutable import metadata: file_name, file_total (sum of every
* transaction's signed amount, the control total), tag (flag term
* name), flag_tid.
* @param array $context
* Batch context (sandbox holds index + mutable quotas, results hold
* the accumulators finished() assembles the summary from).
*/
public static function operation(array $transactions, array $quotas, array $meta, array &$context): void {
if (!isset($context['sandbox']['index'])) {
$context['sandbox']['index'] = 0;
$context['sandbox']['total'] = count($transactions);
$context['sandbox']['quotas'] = $quotas;
// Seed results with the immutable import metadata (no key
// collision with the accumulators) + the zeroed accumulators.
$context['results'] += $meta + [
'created' => 0,
'duplicates' => 0,
'matched' => 0,
'unmatched' => 0,
'created_total' => 0.0,
'duplicates_total' => 0.0,
'duplicates_list' => [],
'errors' => [],
];
}
/** @var \Drupal\figli_compta_ledger\SkipValidationContext $skip */
$skip = \Drupal::service('figli_compta_ledger.skip_validation_context');
/** @var \Drupal\figli_compta_ledger\Import\ClientMatcher $matcher */
$matcher = \Drupal::service('figli_compta_ledger.client_matcher');
$end = min($context['sandbox']['index'] + self::CHUNK, $context['sandbox']['total']);
while ($context['sandbox']['index'] < $end) {
$t = ReleveTransaction::fromArray($transactions[$context['sandbox']['index']]);
// Count-aware dedup: quota exhausted → already in base (this many
// times), skip but surface it on the result page for human review.
if (($context['sandbox']['quotas'][$t->fitid] ?? 0) <= 0) {
$context['results']['duplicates']++;
$context['results']['duplicates_total'] += $t->montant;
$context['results']['duplicates_list'][] = [
'date' => $t->date,
'montant' => $t->montant,
'libelle' => $t->libelle,
];
$context['sandbox']['index']++;
continue;
}
$context['sandbox']['quotas'][$t->fitid]--;
$client = $matcher->match($t->libelle);
// Sensible truncate for the required title field: the full label
// lives in field_notes, the title only backs it up as fallback
// (same libelle display rule as everywhere in /lignes).
$node = Node::create([
'type' => 'ligne_comptable',
'title' => mb_substr($t->libelle, 0, 255),
'uid' => \Drupal::currentUser()->id(),
'status' => 1,
'field_date_ligne' => $t->date,
'field_type_ligne' => 'autre',
// Immutable audit reference (written here, never again), plus
// the three "same value to start with" fields the associate
// corrects while sorting (see PLAN's HT vs TTC section).
'field_montant_releve' => $t->montant,
'field_montant_ht' => $t->montant,
'field_montant_ttc' => $t->montant,
'field_tva' => 0,
'field_notes' => $t->libelle,
'field_import_fitid' => $t->fitid,
'field_client' => $client ? $client->id() : NULL,
'field_flag' => [$meta['flag_tid']],
]);
try {
$skip->skip(fn () => $node->save());
$context['results']['created']++;
$context['results']['created_total'] += $t->montant;
$client ? $context['results']['matched']++ : $context['results']['unmatched']++;
}
catch (EntityStorageException $e) {
$context['results']['errors'][] = [
'libelle' => $t->libelle,
'error' => $e->getMessage(),
];
}
$context['sandbox']['index']++;
}
$context['message'] = t('Import du relevé : @done/@total transactions', [
'@done' => $context['sandbox']['index'],
'@total' => $context['sandbox']['total'],
]);
$context['finished'] = $context['sandbox']['total'] > 0
? $context['sandbox']['index'] / $context['sandbox']['total']
: 1;
}
/**
* Batch finished callback: assembles the summary the result page
* reads -- including the accounting control total (file total must
* equal created + duplicates, to the centime; if not, a parsing bug
* silently ate a line somewhere, and the page says so loudly) -- and
* stores it in the private tempstore (per-user, request-safe), where
* ReleveImportResultController picks it up once and purges it.
*/
public static function finished(bool $success, array $results, array $operations): void {
if (!$success) {
\Drupal::messenger()->addError("L'import a échoué à mi-parcours. Les transactions déjà traitées sont enregistrées ; relancez l'import du même fichier, le dédoublonnage ne recréera que ce qui manque.");
return;
}
$created_total = round((float) ($results['created_total'] ?? 0.0), 2);
$duplicates_total = round((float) ($results['duplicates_total'] ?? 0.0), 2);
$file_total = round((float) ($results['file_total'] ?? 0.0), 2);
$summary = [
'file_name' => (string) ($results['file_name'] ?? ''),
'tag' => (string) ($results['tag'] ?? ''),
'created' => (int) ($results['created'] ?? 0),
'duplicates' => (int) ($results['duplicates'] ?? 0),
'matched' => (int) ($results['matched'] ?? 0),
'unmatched' => (int) ($results['unmatched'] ?? 0),
'errors' => $results['errors'] ?? [],
'duplicates_list' => $results['duplicates_list'] ?? [],
'file_total' => $file_total,
'created_total' => $created_total,
'duplicates_total' => $duplicates_total,
// To the centime: every parsed transaction was either created or
// recognized as already in base. Any drift means a line vanished
// -- never expected, always announced.
'totals_ok' => abs($file_total - $created_total - $duplicates_total) < 0.005,
];
\Drupal::service('tempstore.private')->get('figli_compta_ledger')
->set('releve_import_result', $summary);
}
}
@@ -0,0 +1,47 @@
<?php
namespace Drupal\figli_compta_ledger\Import;
/**
* One bank statement transaction, in the neutral shape every parser
* (CSV today, OFX/CMI maybe later) produces -- the rest of the import
* chain (dedup, client matching, node creation) only ever sees this.
*
* $date: AAAA-MM-JJ (converted from the bank format at parse time).
* $montant: signed, negative = money out (Débit), to the centime.
* $libelle: raw bank label (full, untruncated in CSV), the basis for
* client matching and the line's visible Notes.
* $fitid: dedup fingerprint, see CsvReleveParser::fitid().
*/
final class ReleveTransaction {
public function __construct(
public readonly string $date,
public readonly float $montant,
public readonly string $libelle,
public readonly string $fitid,
) {}
/**
* Plain-array shape for Batch API serialization (operation args and
* sandbox are serialized between requests).
*/
public function toArray(): array {
return [
'date' => $this->date,
'montant' => $this->montant,
'libelle' => $this->libelle,
'fitid' => $this->fitid,
];
}
public static function fromArray(array $values): self {
return new self(
(string) $values['date'],
(float) $values['montant'],
(string) $values['libelle'],
(string) $values['fitid'],
);
}
}
@@ -0,0 +1,64 @@
<?php
namespace Drupal\figli_compta_ledger;
/**
* Request-scoped opt-out of the répartition invariant check in
* figli_compta_ledger_node_presave().
*
* The historical state key ('figli_compta_ledger.skip_validation') is a flag
* shared by every request on the site, stored in the database: while a
* programmatic save (inline type change, bank statement import) holds it,
* a *concurrent* normal form save happening in another PHP-FPM request would
* silently skip validation too -- exactly the kind of hole an integrity
* check must never have. This service lives in the dependency injection
* container of its own request, so a skip here is physically invisible to
* every other request; the depth counter makes nested skips safe and the
* try/finally in skip() guarantees it unwinds on exceptions as well as on
* normal completion.
*
* The state key is still honored by node_presave() for backward compatibility
* with already-shipped migration scripts, but new code (batch imports,
* future migrations) must use this service instead.
*/
final class SkipValidationContext {
/**
* Current skip depth (0 = validation active).
*
* @var int
*/
private int $depth = 0;
/**
* Runs $operation with the répartition-sum check disabled for this
* request only, restoring it afterwards whatever happens.
*
* @param callable $operation
* Typically fn () => $node->save().
*
* @return mixed
* Whatever $operation returns.
*/
public function skip(callable $operation): mixed {
$this->depth++;
try {
return $operation();
}
finally {
$this->depth--;
}
}
/**
* Whether the répartition-sum check is currently disabled for this
* request. Read by figli_compta_ledger_node_presave().
*
* @return bool
* TRUE when a skip() is currently in progress.
*/
public function isSkipped(): bool {
return $this->depth > 0;
}
}