Import de relevé bancaire CSV en libre-service (/lignes/importer-releve)
Chaque transaction du relevé devient une ligne « à trier » : type autre, répartition vide (liseré rouge existant via field_ecart), tag field_flag « IMP AAMMJJ » (liseré ambre + filtre existants), client rapproché par mots (ClientMatcher : séquence contiguë, sinon mot significatif unique -- jamais en sous-chaîne, jamais auto-créé). - src/Import/ : CsvReleveParser (ISO-8859-1 confirmé, en-tête strict, fgetcsv avec escape '' explicite -- dépréciation PHP 8.4, rejets propres avec n° de ligne), ReleveImportBatch (Batch API par lots de 25, dédoublonnage COMPTÉ par empreinte field_import_fitid -- max(0, k−m) importe les vrais doublons légitimes et dédoublonne à travers des fichiers qui se chevauchent --, totaux de contrôle au centime sur la page de résultat, résumé en tempstore privé). - ReleveUploadForm : upload private://releves (fichier conservé + usage, hors de portée du cron), parse en validateForm(), batch, redirection vers la page de résultat. - field_montant_releve : référence bancaire immuable, écrite une fois à l'import et jamais par presave ; affichée en TEXTE sous Montant TTC (widget remplacé par un #type item -- un item ne soumet rien et extractFormValues() saute le champ sans valeur soumise, la valeur survit donc à chaque save) ; masquée sur les lignes sans montant. - SkipValidationContext : contournement du contrôle de répartition requête-scopé (ferme le trou de concurrence de l'ancien state global, AUDIT-2026-09-09 §2.2) -- presave honore le service (state gardé pour compat), updateType/updateField basculent dessus. - Permissions (AUDIT §2.2 priorité 1) : access figli ledger sur toutes les routes du module + autocomplete (RouteSubscriber), import réservé Éditeur/Admin, access content retiré du rôle Authenticated (config/sync re-exportée pour les 4 rôles). - Gin : hook_gin_ignore_sticky_form_actions() -- sans ça, le bouton Importer partait dans la barre sticky du chrome masqué. - install : 8011 champs, 8012 index sur les empreintes, 8013/8014 montant_releve sur le formulaire sous le TTC (poids renumérotés). - /lignes : boutons + Ajouter / Importer / Historique dans le footer sticky (compacts), footer colspan dès la première colonne, badges de signalement qui reviennent à la ligne au lieu de déborder.
This commit is contained in:
@@ -18,6 +18,7 @@ class DashboardController extends ControllerBase {
|
||||
return [
|
||||
'#theme' => 'figli_compta_home',
|
||||
'#can_view_history' => $this->currentUser()->hasPermission('view ligne_comptable revisions'),
|
||||
'#can_import_releve' => $this->currentUser()->hasPermission('import ligne_comptable releve'),
|
||||
'#current_route' => 'figli_compta_ledger.home',
|
||||
'#attached' => [
|
||||
'library' => ['figli_compta_ledger/home'],
|
||||
|
||||
@@ -5,8 +5,10 @@ namespace Drupal\figli_compta_ledger\Controller;
|
||||
use Drupal\Core\Access\CsrfRequestHeaderAccessCheck;
|
||||
use Drupal\Core\Controller\ControllerBase;
|
||||
use Drupal\Core\Entity\EntityStorageException;
|
||||
use Drupal\figli_compta_ledger\SkipValidationContext;
|
||||
use Drupal\node\NodeInterface;
|
||||
use Drupal\taxonomy\Entity\Term;
|
||||
use Symfony\Component\DependencyInjection\ContainerInterface;
|
||||
use Symfony\Component\HttpFoundation\JsonResponse;
|
||||
use Symfony\Component\HttpFoundation\Request;
|
||||
|
||||
@@ -23,6 +25,24 @@ use Symfony\Component\HttpFoundation\Request;
|
||||
*/
|
||||
class LedgerActionsController extends ControllerBase {
|
||||
|
||||
/**
|
||||
* Request-scoped répartition-check opt-out -- see the class docblock of
|
||||
* \Drupal\figli_compta_ledger\SkipValidationContext for why this replaced
|
||||
* the historical global state key here.
|
||||
*
|
||||
* @var \Drupal\figli_compta_ledger\SkipValidationContext
|
||||
*/
|
||||
protected $skipValidationContext;
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public static function create(ContainerInterface $container) {
|
||||
$instance = parent::create($container);
|
||||
$instance->skipValidationContext = $container->get('figli_compta_ledger.skip_validation_context');
|
||||
return $instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fields editable inline from /lignes without opening the full node
|
||||
* edit form -- keys are the short names the frontend sends; values are
|
||||
@@ -96,20 +116,17 @@ class LedgerActionsController extends ControllerBase {
|
||||
// (from historical data, never corrected -- see figli_compta_ledger's
|
||||
// module docblock) exactly as it was. The présave check exists to
|
||||
// catch new inconsistent entries, not to block relabeling the type of
|
||||
// an already-migrated line, so skip it for this save only. try/finally
|
||||
// guarantees the global flag clears even if save() throws for an
|
||||
// unrelated reason -- leaving it on would silently skip validation on
|
||||
// every other save on the site.
|
||||
\Drupal::state()->set('figli_compta_ledger.skip_validation', TRUE);
|
||||
// an already-migrated line, so skip it for this save only. The
|
||||
// SkipValidationContext service is request-scoped with a try/finally
|
||||
// inside skip(), so the check is back on the instant save() returns
|
||||
// or throws -- no global flag left hanging that a concurrent save
|
||||
// from someone else could fall into.
|
||||
try {
|
||||
$node->save();
|
||||
$this->skipValidationContext->skip(fn () => $node->save());
|
||||
}
|
||||
catch (EntityStorageException $e) {
|
||||
return new JsonResponse(['error' => $e->getMessage()], 422);
|
||||
}
|
||||
finally {
|
||||
\Drupal::state()->delete('figli_compta_ledger.skip_validation');
|
||||
}
|
||||
|
||||
return new JsonResponse([
|
||||
'success' => TRUE,
|
||||
@@ -167,17 +184,14 @@ class LedgerActionsController extends ControllerBase {
|
||||
// signalement changes here, montant_ht and field_repartition are
|
||||
// untouched, so skipping the répartition check for this save can
|
||||
// never introduce a mismatch -- it can only leave a pre-existing
|
||||
// historical one exactly as it was.
|
||||
\Drupal::state()->set('figli_compta_ledger.skip_validation', TRUE);
|
||||
// historical one exactly as it was. Request-scoped skip (see
|
||||
// updateType()'s comment), no global flag.
|
||||
try {
|
||||
$node->save();
|
||||
$this->skipValidationContext->skip(fn () => $node->save());
|
||||
}
|
||||
catch (EntityStorageException $e) {
|
||||
return new JsonResponse(['error' => $e->getMessage()], 422);
|
||||
}
|
||||
finally {
|
||||
\Drupal::state()->delete('figli_compta_ledger.skip_validation');
|
||||
}
|
||||
|
||||
if ($field === 'client') {
|
||||
$newValue = $node->get('field_client')->entity ? $node->get('field_client')->entity->label() : NULL;
|
||||
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
<?php
|
||||
|
||||
namespace Drupal\figli_compta_ledger\Controller;
|
||||
|
||||
use Drupal\Core\Controller\ControllerBase;
|
||||
use Drupal\Core\Routing\LocalRedirectResponse;
|
||||
use Drupal\Core\Url;
|
||||
|
||||
/**
|
||||
* Post-import summary page: counts, the accounting control total
|
||||
* (file total == created + duplicates, to the centime), the list of
|
||||
* duplicates skipped for human review, and a deep link into /lignes
|
||||
* filtered on this batch's "IMP AAMMJJ" tag.
|
||||
*
|
||||
* The numbers live one-shot in the private tempstore (written by
|
||||
* ReleveImportBatch::finished(), read + purged here): the page is only
|
||||
* meaningful right after an import, and every user sees their own.
|
||||
*/
|
||||
class ReleveImportResultController extends ControllerBase {
|
||||
|
||||
/**
|
||||
* Renders the summary of the import that just ran.
|
||||
*/
|
||||
public function result() {
|
||||
$store = \Drupal::service('tempstore.private')->get('figli_compta_ledger');
|
||||
$summary = $store->get('releve_import_result');
|
||||
if (!$summary) {
|
||||
// Direct navigation (bookmark, back button long after the import):
|
||||
// no numbers in memory anymore, send back to the form instead of
|
||||
// showing an empty shell.
|
||||
$this->messenger()->addWarning($this->t("Le résultat d'un import n'est disponible qu'immédiatement après l'import lui-même."));
|
||||
return new LocalRedirectResponse(Url::fromRoute('figli_compta_ledger.releve_import_form')->toString());
|
||||
}
|
||||
$store->delete('releve_import_result');
|
||||
|
||||
$eur = fn ($x) => number_format((float) $x, 2, ',', ' ') . ' €';
|
||||
$fr_date = fn ($iso) => preg_replace('/^(\d{4})-(\d{2})-(\d{2})$/', '$3/$2/$1', (string) $iso);
|
||||
|
||||
$view = [
|
||||
'file_name' => $summary['file_name'],
|
||||
'tag' => $summary['tag'],
|
||||
'created' => (int) $summary['created'],
|
||||
'duplicates' => (int) $summary['duplicates'],
|
||||
'matched' => (int) $summary['matched'],
|
||||
'unmatched' => (int) $summary['unmatched'],
|
||||
'errors_count' => count($summary['errors']),
|
||||
'errors' => array_map(fn ($e) => $e['libelle'] . ' — ' . $e['error'], $summary['errors']),
|
||||
'duplicates_list' => array_map(fn ($d) => [
|
||||
'date' => $fr_date($d['date']),
|
||||
'montant' => $eur($d['montant']),
|
||||
'libelle' => $d['libelle'],
|
||||
], $summary['duplicates_list']),
|
||||
'file_total' => $eur($summary['file_total']),
|
||||
'created_total' => $eur($summary['created_total']),
|
||||
'duplicates_total' => $eur($summary['duplicates_total']),
|
||||
'totals_ok' => (bool) $summary['totals_ok'],
|
||||
];
|
||||
|
||||
return [
|
||||
'#theme' => 'figli_compta_releve_import_result',
|
||||
'#summary' => $view,
|
||||
// /lignes reads its filter state from location.hash -- the flag
|
||||
// filter key is "tag" (see readHashState() in js/home.js), values
|
||||
// are comma-separated flag names.
|
||||
'#lignes_url' => Url::fromRoute('figli_compta_ledger.home')->toString() . '#tag=' . rawurlencode($summary['tag']),
|
||||
'#import_url' => Url::fromRoute('figli_compta_ledger.releve_import_form')->toString(),
|
||||
'#attached' => ['library' => ['figli_compta_ledger/releve_import']],
|
||||
'#cache' => ['max-age' => 0],
|
||||
];
|
||||
}
|
||||
|
||||
}
|
||||
@@ -24,7 +24,11 @@ class RouteSubscriber extends RouteSubscriberBase {
|
||||
*/
|
||||
protected function alterRoutes(RouteCollection $collection) {
|
||||
if ($route = $collection->get('system.entity_autocomplete')) {
|
||||
$route->setRequirement('_permission', 'access content');
|
||||
// 'access figli ledger' rather than 'access content': the generic
|
||||
// Authenticated role no longer holds the latter (removed 2026-09),
|
||||
// and anyone entitled to see ledger autocomplete suggestions must
|
||||
// be entitled to the ledger's data itself.
|
||||
$route->setRequirement('_permission', 'access figli ledger');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
<?php
|
||||
|
||||
namespace Drupal\figli_compta_ledger\Form;
|
||||
|
||||
use Drupal\Core\Form\FormBase;
|
||||
use Drupal\Core\Form\FormStateInterface;
|
||||
use Drupal\file\Entity\File;
|
||||
use Drupal\figli_compta_ledger\Import\CsvReleveParser;
|
||||
use Drupal\figli_compta_ledger\Import\ReleveImportBatch;
|
||||
use Drupal\figli_compta_ledger\Import\ReleveTransaction;
|
||||
use Drupal\taxonomy\Entity\Term;
|
||||
|
||||
/**
|
||||
* Upload form for a bank statement export (CSV v1 -- see
|
||||
* PLAN-import-releve-bancaire.md). Self-service for the associates: the
|
||||
* file lands in the *private* filesystem (financial data), is parsed in
|
||||
* memory, deduplicated count-aware against lines already in base, then
|
||||
* turned into "à trier" draft lines by ReleveImportBatch. Nothing here
|
||||
* writes ledger lines directly -- everything goes through the normal
|
||||
* Node::save() lifecycle under SkipValidationContext.
|
||||
*/
|
||||
class ReleveUploadForm extends FormBase {
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function getFormId(): string {
|
||||
return 'figli_compta_ledger_releve_upload_form';
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function buildForm(array $form, FormStateInterface $form_state): array {
|
||||
$form['releve_file'] = [
|
||||
'#type' => 'managed_file',
|
||||
'#title' => $this->t('Relevé bancaire (CSV)'),
|
||||
'#upload_location' => 'private://releves',
|
||||
// Only .csv: the OFX/CMI exports of the same account exist but
|
||||
// are explicitly out of v1 scope (truncated labels / unstable
|
||||
// structure -- see the plan). A clear message beats a silent
|
||||
// failure for someone uploading them by mistake.
|
||||
'#upload_validators' => [
|
||||
'FileExtension' => ['extensions' => 'csv'],
|
||||
],
|
||||
'#required' => TRUE,
|
||||
'#description' => $this->t('Export CSV de la banque : colonnes « Date ; Date de valeur ; Débit ; Crédit ; Libellé ; Solde » (les fichiers .ofx et .cmi ne sont pas pris en charge pour l\'instant). Chaque transaction devient une ligne « à trier » : type, répartition et HT/TVA restent à assigner à la main.'),
|
||||
];
|
||||
|
||||
$form['actions'] = ['#type' => 'actions'];
|
||||
$form['actions']['submit'] = [
|
||||
'#type' => 'submit',
|
||||
'#value' => $this->t('Importer le relevé'),
|
||||
'#button_type' => 'primary',
|
||||
];
|
||||
return $form;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*
|
||||
* The idiomatic home for the parse: an invalid file is a validation
|
||||
* error, rejected before anything is written (the file entity is only
|
||||
* promoted in submitForm()). The parsed transactions are stashed in
|
||||
* $form_state so the file is never parsed twice.
|
||||
*/
|
||||
public function validateForm(array &$form, FormStateInterface $form_state): void {
|
||||
$fids = $form_state->getValue('releve_file');
|
||||
$fids = is_array($fids) ? $fids : [];
|
||||
if (!$fids) {
|
||||
// #required already covers the empty case.
|
||||
return;
|
||||
}
|
||||
$file = File::load(reset($fids));
|
||||
if (!$file) {
|
||||
$form_state->setErrorByName('releve_file', $this->t("Le fichier téléversé n'a pas pu être retrouvé."));
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse (pure, no writes). A clean form error -- never a crash page
|
||||
// -- for anything the parser rejects.
|
||||
try {
|
||||
$transactions = (new CsvReleveParser())->parse($file->getFileUri());
|
||||
}
|
||||
catch (\RuntimeException $e) {
|
||||
$form_state->setErrorByName('releve_file', $e->getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
$form_state->set('releve_fid', (int) $file->id());
|
||||
$form_state->set('releve_transactions', array_map(fn ($t) => $t->toArray(), $transactions));
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function submitForm(array &$form, FormStateInterface $form_state): void {
|
||||
$file = File::load($form_state->get('releve_fid'));
|
||||
$transactions = array_map([ReleveTransaction::class, 'fromArray'], $form_state->get('releve_transactions') ?: []);
|
||||
if (!$file || !$transactions) {
|
||||
// Normally unreachable -- validateForm() blocks bad files before
|
||||
// submit is reached. Defensive only.
|
||||
$form_state->setErrorByName('releve_file', $this->t("Rien à importer : relancez l'upload."));
|
||||
return;
|
||||
}
|
||||
|
||||
// Keep the uploaded statement permanently + registered as our usage:
|
||||
// it's accounting source material, cron must not garbage-collect it
|
||||
// after a few hours as it would a temporary file.
|
||||
$file->setPermanent();
|
||||
$file->save();
|
||||
\Drupal::service('file.usage')->add($file, 'figli_compta_ledger', 'releve_import', (int) $file->id());
|
||||
|
||||
// Count-aware dedup: how many times each fingerprint appears in this
|
||||
// file (k), one grouped query for how many already exist in base
|
||||
// (m, all provenances combined), quota = max(0, k − m).
|
||||
$counts = [];
|
||||
foreach ($transactions as $t) {
|
||||
$counts[$t->fitid] = ($counts[$t->fitid] ?? 0) + 1;
|
||||
}
|
||||
$db_counts = [];
|
||||
if ($counts) {
|
||||
$select = \Drupal::database()->select('node__field_import_fitid', 'f')
|
||||
->condition('f.field_import_fitid_value', array_keys($counts), 'IN');
|
||||
$select->addField('f', 'field_import_fitid_value', 'fitid');
|
||||
$select->addExpression('COUNT(*)', 'n');
|
||||
$select->groupBy('f.field_import_fitid_value');
|
||||
foreach ($select->execute()->fetchAllKeyed() as $fitid => $n) {
|
||||
$db_counts[$fitid] = (int) $n;
|
||||
}
|
||||
}
|
||||
$quotas = [];
|
||||
foreach ($counts as $fitid => $k) {
|
||||
$quotas[$fitid] = max(0, $k - ($db_counts[$fitid] ?? 0));
|
||||
}
|
||||
|
||||
// One "IMP AAMMJJ" flag term per import batch (day granularity: the
|
||||
// same day's re-imports join the same lot) -- short on purpose, it
|
||||
// renders as a badge in /lignes' narrow Signalement column. The
|
||||
// associates sort lines through the existing signalement mechanism
|
||||
// (filter + amber marker), zero new UI.
|
||||
$tag = 'IMP ' . date('ymd');
|
||||
$terms = \Drupal::entityTypeManager()->getStorage('taxonomy_term')
|
||||
->loadByProperties(['vid' => 'flag', 'name' => $tag]);
|
||||
if ($terms) {
|
||||
$term = reset($terms);
|
||||
}
|
||||
else {
|
||||
$term = Term::create(['vid' => 'flag', 'name' => $tag]);
|
||||
$term->save();
|
||||
}
|
||||
|
||||
$file_total = 0.0;
|
||||
$payload = [];
|
||||
foreach ($transactions as $t) {
|
||||
$file_total += $t->montant;
|
||||
$payload[] = $t->toArray();
|
||||
}
|
||||
|
||||
batch_set([
|
||||
'title' => $this->t('Import du relevé bancaire'),
|
||||
'operations' => [
|
||||
[
|
||||
[ReleveImportBatch::class, 'operation'],
|
||||
[$payload, $quotas, [
|
||||
'file_name' => $file->getFilename(),
|
||||
'file_total' => round($file_total, 2),
|
||||
'tag' => $tag,
|
||||
'flag_tid' => (int) $term->id(),
|
||||
]],
|
||||
],
|
||||
],
|
||||
'finished' => [ReleveImportBatch::class, 'finished'],
|
||||
'init_message' => $this->t('Import du relevé en cours…'),
|
||||
'progress_message' => $this->t('@current/@total'),
|
||||
'error_message' => $this->t('L\'import a rencontré une erreur inattendue.'),
|
||||
]);
|
||||
|
||||
// Where the browser lands once the batch is done -- the result page
|
||||
// reads its numbers from the private tempstore.
|
||||
$form_state->setRedirect('figli_compta_ledger.releve_import_result');
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
<?php
|
||||
|
||||
namespace Drupal\figli_compta_ledger\Import;
|
||||
|
||||
use Drupal\taxonomy\Entity\Term;
|
||||
|
||||
/**
|
||||
* Suggests which client term a bank label refers to -- conservatively:
|
||||
* an empty match a human fills in beats a wrong match nobody re-checks
|
||||
* (the same philosophy as the whole import feature: pre-fill, never
|
||||
* decide).
|
||||
*
|
||||
* Matching is word-based, never raw substring: after normalization
|
||||
* (uppercase, accents removed, non-alphanumerics as separators) a
|
||||
* client's full name must appear as a contiguous word *sequence* in the
|
||||
* label ("OVH SAS" matches "PRLV SEPA OVH SAS TWLN…" but a hypothetical
|
||||
* client "AIR" would NOT match "CLAIR" -- the v0 substring draft of this
|
||||
* plan had exactly that false-positive mode for short names).
|
||||
*
|
||||
* Pass 1: full normalized client name as contiguous word sequence. Two
|
||||
* distinct clients matching is ambiguous → empty.
|
||||
* Pass 2 (only if pass 1 found nothing): a single "significant" word
|
||||
* (≥ 4 chars, not a legal-form filler like SAS/SARL) that belongs to
|
||||
* exactly ONE client in the whole vocabulary. Several candidate clients
|
||||
* → empty. Deliberately recall-biased: a generic-but-unique word (say
|
||||
* "MAISON", held by a single client) can suggest the wrong client for
|
||||
* an unrelated label -- acceptable because every imported line is
|
||||
* flagged and manually sorted (see ReleveImportBatch), so a wrong
|
||||
* suggestion gets corrected by a human rather than trusted.
|
||||
*/
|
||||
final class ClientMatcher {
|
||||
|
||||
/**
|
||||
* Legal-form filler words never significant on their own.
|
||||
*/
|
||||
private const STOPWORDS = ['SAS', 'SARL', 'SA', 'SASU', 'EURL', 'SCI', 'SCOP', 'ASSOCIATION', 'GMBH', 'SNC'];
|
||||
|
||||
/**
|
||||
* Loaded client vocabulary, shape: [['term' => Term, 'words' => string[]]].
|
||||
*
|
||||
* @var array|null
|
||||
*/
|
||||
private ?array $clients = NULL;
|
||||
|
||||
/**
|
||||
* Returns the client term a bank label most likely refers to, or NULL
|
||||
* when nothing safe can be said. Never creates a term (unlike flag
|
||||
* auto-creation) -- the client vocabulary stays curated by hand.
|
||||
*/
|
||||
public function match(string $libelle): ?Term {
|
||||
$words = $this->words($libelle);
|
||||
if (!$words) {
|
||||
return NULL;
|
||||
}
|
||||
$clients = $this->loadClients();
|
||||
if (!$clients) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Pass 1: full name as a contiguous word sequence, unique candidate.
|
||||
$pass1 = [];
|
||||
foreach ($clients as $client) {
|
||||
if (self::containsSequence($words, $client['words'])) {
|
||||
$pass1[$client['term']->id()] = $client['term'];
|
||||
}
|
||||
}
|
||||
if (count($pass1) === 1) {
|
||||
return reset($pass1);
|
||||
}
|
||||
if (count($pass1) > 1) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Pass 2: a significant word owned by exactly one client vocabulary.
|
||||
$pass2 = [];
|
||||
foreach ($clients as $client) {
|
||||
foreach ($client['words'] as $word) {
|
||||
if (mb_strlen($word) < 4 || in_array($word, self::STOPWORDS, TRUE)) {
|
||||
continue;
|
||||
}
|
||||
if (in_array($word, $words, TRUE)) {
|
||||
$pass2[$client['term']->id()] = $client['term'];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (count($pass2) === 1) {
|
||||
return reset($pass2);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Uppercase, accent-free word tokens: "EPAU / POPSU" → ["EPAU","POPSU"].
|
||||
*
|
||||
* @return string[]
|
||||
*/
|
||||
private function words(string $text): array {
|
||||
$transliterated = \Drupal::transliteration()->transliterate($text, 'fr');
|
||||
$upper = mb_strtoupper($transliterated);
|
||||
$words = preg_split('/[^A-Z0-9]+/', $upper, -1, PREG_SPLIT_NO_EMPTY);
|
||||
return $words === FALSE ? [] : $words;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads (once per request) every client term with its normalized words.
|
||||
*/
|
||||
private function loadClients(): array {
|
||||
if ($this->clients !== NULL) {
|
||||
return $this->clients;
|
||||
}
|
||||
$this->clients = [];
|
||||
$terms = \Drupal::entityTypeManager()->getStorage('taxonomy_term')
|
||||
->loadByProperties(['vid' => 'client']);
|
||||
foreach ($terms as $term) {
|
||||
$words = $this->words($term->label());
|
||||
if ($words) {
|
||||
$this->clients[] = ['term' => $term, 'words' => $words];
|
||||
}
|
||||
}
|
||||
return $this->clients;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether $needle appears in $haystack as a contiguous word sequence.
|
||||
*/
|
||||
private static function containsSequence(array $haystack, array $needle): bool {
|
||||
$n = count($needle);
|
||||
$h = count($haystack);
|
||||
if ($n === 0 || $n > $h) {
|
||||
return FALSE;
|
||||
}
|
||||
for ($i = 0; $i <= $h - $n; $i++) {
|
||||
for ($j = 0; $j < $n; $j++) {
|
||||
if ($haystack[$i + $j] !== $needle[$j]) {
|
||||
continue 2;
|
||||
}
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
<?php
|
||||
|
||||
namespace Drupal\figli_compta_ledger\Import;
|
||||
|
||||
/**
|
||||
* Parses the CSV export of the SAS bank account into ReleveTransaction
|
||||
* objects. Built from (and verified against) the real sample in
|
||||
* sources-compta/extrais de comptes/00021322002.csv: `;`-separated,
|
||||
* ISO-8859-1 encoded, columns "Date;Date de valeur;Débit;Crédit;Libellé;
|
||||
* Solde", dates JJ/MM/AAAA, French decimal comma, exactly one of
|
||||
* Débit/Crédit filled per row. The Solde column is ignored (no balance
|
||||
* reconciliation in v1).
|
||||
*
|
||||
* Any deviation (wrong header, unparsable date/amount, empty label) throws
|
||||
* a RuntimeException with a clear, user-facing French message -- the upload
|
||||
* form catches it and shows a form error, never a raw crash page. Nothing
|
||||
* is written to the database from here: parsing is a pure in-memory step.
|
||||
*/
|
||||
final class CsvReleveParser {
|
||||
|
||||
/**
|
||||
* The exact header (after ISO-8859-1 → UTF-8 conversion) a file must
|
||||
* carry to be considered a supported statement export.
|
||||
*/
|
||||
private const HEADER = ['Date', 'Date de valeur', 'Débit', 'Crédit', 'Libellé', 'Solde'];
|
||||
|
||||
/**
|
||||
* Parses a file by URI (any stream wrapper, typically private://).
|
||||
*
|
||||
* @return \Drupal\figli_compta_ledger\Import\ReleveTransaction[]
|
||||
* Every data row as a transaction, in file order.
|
||||
*
|
||||
* @throws \RuntimeException
|
||||
* With a ready-to-display message when the file isn't a supported
|
||||
* statement export.
|
||||
*/
|
||||
public function parse(string $uri): array {
|
||||
$stream = @fopen($uri, 'r');
|
||||
if ($stream === FALSE) {
|
||||
throw new \RuntimeException("Le fichier téléversé n'a pas pu être relu depuis le stockage privé.");
|
||||
}
|
||||
// Confirmed ISO-8859-1 on the real sample: a naive UTF-8 read would
|
||||
// corrupt every accented label -- the one truly silent bug risk of
|
||||
// this parser. The filter converts as fgetcsv() reads.
|
||||
stream_filter_append($stream, 'convert.iconv.ISO-8859-1/UTF-8');
|
||||
|
||||
// Explicit enclosure + empty $escape: PHP 8.4 deprecates relying on
|
||||
// the default escape (backslash), whose legacy behavior would let a
|
||||
// stray "\" in a bank label swallow the next character -- with ''
|
||||
// the bank's own quotes stay the only special characters.
|
||||
$header = fgetcsv($stream, NULL, ';', '"', '');
|
||||
if ($header === FALSE) {
|
||||
fclose($stream);
|
||||
throw new \RuntimeException('Le fichier est vide.');
|
||||
}
|
||||
$header = array_map(fn ($h) => trim((string) $h), $header);
|
||||
if ($header !== self::HEADER) {
|
||||
fclose($stream);
|
||||
throw new \RuntimeException('Format de fichier non reconnu. En-tête attendu : « ' . implode(';', self::HEADER) . ' ». Seul l\'export CSV de la banque est pris en charge pour l\'instant (.ofx et .cmi non encore).');
|
||||
}
|
||||
|
||||
$transactions = [];
|
||||
$line = 1;
|
||||
while (($row = fgetcsv($stream, NULL, ';', '"', '')) !== FALSE) {
|
||||
$line++;
|
||||
// Fully blank rows are just padding at the end of some exports.
|
||||
if (trim(implode('', array_map('strval', $row))) === '') {
|
||||
continue;
|
||||
}
|
||||
if (count($row) < 6) {
|
||||
fclose($stream);
|
||||
throw new \RuntimeException("Ligne $line : nombre de colonnes inattendu (" . count($row) . ", 6 attendues).");
|
||||
}
|
||||
|
||||
$dateRaw = trim((string) $row[0]);
|
||||
if (!preg_match('/^(\d{2})\/(\d{2})\/(\d{4})$/', $dateRaw, $m) || !checkdate((int) $m[2], (int) $m[1], (int) $m[3])) {
|
||||
fclose($stream);
|
||||
throw new \RuntimeException("Ligne $line : date « $dateRaw » invalide (JJ/MM/AAAA attendu).");
|
||||
}
|
||||
$date = $m[3] . '-' . $m[2] . '-' . $m[1];
|
||||
|
||||
$debit = trim((string) $row[2]);
|
||||
$credit = trim((string) $row[3]);
|
||||
if ($debit !== '' && $credit !== '') {
|
||||
fclose($stream);
|
||||
throw new \RuntimeException("Ligne $line : Débit et Crédit renseignés simultanément, format inattendu.");
|
||||
}
|
||||
// A Débit is money out whatever sign the bank exported it with
|
||||
// (the sample already stores it negative; -abs() normalizes any
|
||||
// sibling export that doesn't), a Crédit is money in.
|
||||
if ($debit !== '') {
|
||||
$montant = -abs($this->parseAmount($debit, $line));
|
||||
}
|
||||
elseif ($credit !== '') {
|
||||
$montant = abs($this->parseAmount($credit, $line));
|
||||
}
|
||||
else {
|
||||
fclose($stream);
|
||||
throw new \RuntimeException("Ligne $line : ni Débit ni Crédit renseigné.");
|
||||
}
|
||||
|
||||
$libelle = self::normalizeLibelle((string) $row[4]);
|
||||
if ($libelle === '') {
|
||||
fclose($stream);
|
||||
throw new \RuntimeException("Ligne $line : libellé vide, impossible de tracer la transaction.");
|
||||
}
|
||||
|
||||
$transactions[] = new ReleveTransaction(
|
||||
$date,
|
||||
$montant,
|
||||
$libelle,
|
||||
self::fitid($date, $montant, $libelle),
|
||||
);
|
||||
}
|
||||
fclose($stream);
|
||||
|
||||
if (!$transactions) {
|
||||
throw new \RuntimeException("Aucune transaction trouvée dans le fichier (en-tête seul).");
|
||||
}
|
||||
return $transactions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fingerprint of one transaction: 'csv:' + sha1(date | signed amount to
|
||||
* the centime | whitespace-normalized label). No case-folding -- two
|
||||
* exports of the same account reproduce labels byte for byte, and the
|
||||
* fingerprint must stay stable for the count-aware dedup to recognize
|
||||
* an already-imported transaction years later.
|
||||
*
|
||||
* The amount goes in as a fixed 2-decimal string ("−1234.56") so no
|
||||
* floating-point representation ever enters the hash.
|
||||
*/
|
||||
public static function fitid(string $date, float $montant, string $normalizedLibelle): string {
|
||||
return 'csv:' . sha1($date . '|' . number_format($montant, 2, '.', '') . '|' . $normalizedLibelle);
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim + collapse internal whitespace runs to one space: stray double
|
||||
* spaces would otherwise make the same transaction fingerprint
|
||||
* differently across two exports of the same account.
|
||||
*/
|
||||
public static function normalizeLibelle(string $libelle): string {
|
||||
return trim((string) preg_replace('/\s+/u', ' ', $libelle));
|
||||
}
|
||||
|
||||
/**
|
||||
* French decimal ("1 234,56", "-45,89") → float, with a hard format
|
||||
* check -- anything unexpected rejects the whole file with the line
|
||||
* number rather than being silently coerced.
|
||||
*/
|
||||
private function parseAmount(string $raw, int $line): float {
|
||||
$clean = str_replace([' ', "\xC2\xA0"], '', $raw);
|
||||
$clean = str_replace(',', '.', $clean);
|
||||
if (!preg_match('/^[+-]?\d+(\.\d+)?$/', $clean)) {
|
||||
throw new \RuntimeException("Ligne $line : montant « $raw » invalide.");
|
||||
}
|
||||
return (float) $clean;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
<?php
|
||||
|
||||
namespace Drupal\figli_compta_ledger\Import;
|
||||
|
||||
use Drupal\Core\Entity\EntityStorageException;
|
||||
use Drupal\node\Entity\Node;
|
||||
use Drupal\taxonomy\Entity\Term;
|
||||
|
||||
/**
|
||||
* Batch backend of the bank statement import: turns the parsed
|
||||
* transactions into ligne_comptable nodes, ~25 per PHP-FPM request
|
||||
* (several hundred transactions would blow the memory/time budget of a
|
||||
* single request -- no Batch API precedent existed in this module
|
||||
* before this).
|
||||
*
|
||||
* Count-aware dedup (see PLAN-import-releve-bancaire.md): for every
|
||||
* fingerprint, the file tells how many times the transaction appears (k)
|
||||
* and the database how many are already imported (m) -- the batch then
|
||||
* creates max(0, k − m) lines. That imports every legitimate duplicate
|
||||
* (two identical transfers the same day) while still recognizing an
|
||||
* already-imported transaction across overlapping files.
|
||||
*
|
||||
* Every created line goes through SkipValidationContext (request-scoped,
|
||||
* NOT the historical state key): répartition is deliberately empty, so
|
||||
* the presave invariant sum(répartition) == montant_ht must not fire --
|
||||
* field_ecart (= montant_ht) still gets computed and drives the existing
|
||||
* "à trier" red marker on /lignes.
|
||||
*/
|
||||
final class ReleveImportBatch {
|
||||
|
||||
/**
|
||||
* Transactions processed per batch step.
|
||||
*/
|
||||
const CHUNK = 25;
|
||||
|
||||
/**
|
||||
* Batch operation -- called repeatedly by Drupal until finished.
|
||||
*
|
||||
* @param array $transactions
|
||||
* ReleveTransaction::toArray() payloads, in file order.
|
||||
* @param array $quotas
|
||||
* fitid => remaining lines to create (k − m, floored at 0).
|
||||
* @param array $meta
|
||||
* Immutable import metadata: file_name, file_total (sum of every
|
||||
* transaction's signed amount, the control total), tag (flag term
|
||||
* name), flag_tid.
|
||||
* @param array $context
|
||||
* Batch context (sandbox holds index + mutable quotas, results hold
|
||||
* the accumulators finished() assembles the summary from).
|
||||
*/
|
||||
public static function operation(array $transactions, array $quotas, array $meta, array &$context): void {
|
||||
if (!isset($context['sandbox']['index'])) {
|
||||
$context['sandbox']['index'] = 0;
|
||||
$context['sandbox']['total'] = count($transactions);
|
||||
$context['sandbox']['quotas'] = $quotas;
|
||||
// Seed results with the immutable import metadata (no key
|
||||
// collision with the accumulators) + the zeroed accumulators.
|
||||
$context['results'] += $meta + [
|
||||
'created' => 0,
|
||||
'duplicates' => 0,
|
||||
'matched' => 0,
|
||||
'unmatched' => 0,
|
||||
'created_total' => 0.0,
|
||||
'duplicates_total' => 0.0,
|
||||
'duplicates_list' => [],
|
||||
'errors' => [],
|
||||
];
|
||||
}
|
||||
|
||||
/** @var \Drupal\figli_compta_ledger\SkipValidationContext $skip */
|
||||
$skip = \Drupal::service('figli_compta_ledger.skip_validation_context');
|
||||
/** @var \Drupal\figli_compta_ledger\Import\ClientMatcher $matcher */
|
||||
$matcher = \Drupal::service('figli_compta_ledger.client_matcher');
|
||||
|
||||
$end = min($context['sandbox']['index'] + self::CHUNK, $context['sandbox']['total']);
|
||||
while ($context['sandbox']['index'] < $end) {
|
||||
$t = ReleveTransaction::fromArray($transactions[$context['sandbox']['index']]);
|
||||
|
||||
// Count-aware dedup: quota exhausted → already in base (this many
|
||||
// times), skip but surface it on the result page for human review.
|
||||
if (($context['sandbox']['quotas'][$t->fitid] ?? 0) <= 0) {
|
||||
$context['results']['duplicates']++;
|
||||
$context['results']['duplicates_total'] += $t->montant;
|
||||
$context['results']['duplicates_list'][] = [
|
||||
'date' => $t->date,
|
||||
'montant' => $t->montant,
|
||||
'libelle' => $t->libelle,
|
||||
];
|
||||
$context['sandbox']['index']++;
|
||||
continue;
|
||||
}
|
||||
$context['sandbox']['quotas'][$t->fitid]--;
|
||||
|
||||
$client = $matcher->match($t->libelle);
|
||||
// Sensible truncate for the required title field: the full label
|
||||
// lives in field_notes, the title only backs it up as fallback
|
||||
// (same libelle display rule as everywhere in /lignes).
|
||||
$node = Node::create([
|
||||
'type' => 'ligne_comptable',
|
||||
'title' => mb_substr($t->libelle, 0, 255),
|
||||
'uid' => \Drupal::currentUser()->id(),
|
||||
'status' => 1,
|
||||
'field_date_ligne' => $t->date,
|
||||
'field_type_ligne' => 'autre',
|
||||
// Immutable audit reference (written here, never again), plus
|
||||
// the three "same value to start with" fields the associate
|
||||
// corrects while sorting (see PLAN's HT vs TTC section).
|
||||
'field_montant_releve' => $t->montant,
|
||||
'field_montant_ht' => $t->montant,
|
||||
'field_montant_ttc' => $t->montant,
|
||||
'field_tva' => 0,
|
||||
'field_notes' => $t->libelle,
|
||||
'field_import_fitid' => $t->fitid,
|
||||
'field_client' => $client ? $client->id() : NULL,
|
||||
'field_flag' => [$meta['flag_tid']],
|
||||
]);
|
||||
try {
|
||||
$skip->skip(fn () => $node->save());
|
||||
$context['results']['created']++;
|
||||
$context['results']['created_total'] += $t->montant;
|
||||
$client ? $context['results']['matched']++ : $context['results']['unmatched']++;
|
||||
}
|
||||
catch (EntityStorageException $e) {
|
||||
$context['results']['errors'][] = [
|
||||
'libelle' => $t->libelle,
|
||||
'error' => $e->getMessage(),
|
||||
];
|
||||
}
|
||||
$context['sandbox']['index']++;
|
||||
}
|
||||
|
||||
$context['message'] = t('Import du relevé : @done/@total transactions', [
|
||||
'@done' => $context['sandbox']['index'],
|
||||
'@total' => $context['sandbox']['total'],
|
||||
]);
|
||||
$context['finished'] = $context['sandbox']['total'] > 0
|
||||
? $context['sandbox']['index'] / $context['sandbox']['total']
|
||||
: 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch finished callback: assembles the summary the result page
|
||||
* reads -- including the accounting control total (file total must
|
||||
* equal created + duplicates, to the centime; if not, a parsing bug
|
||||
* silently ate a line somewhere, and the page says so loudly) -- and
|
||||
* stores it in the private tempstore (per-user, request-safe), where
|
||||
* ReleveImportResultController picks it up once and purges it.
|
||||
*/
|
||||
public static function finished(bool $success, array $results, array $operations): void {
|
||||
if (!$success) {
|
||||
\Drupal::messenger()->addError("L'import a échoué à mi-parcours. Les transactions déjà traitées sont enregistrées ; relancez l'import du même fichier, le dédoublonnage ne recréera que ce qui manque.");
|
||||
return;
|
||||
}
|
||||
|
||||
$created_total = round((float) ($results['created_total'] ?? 0.0), 2);
|
||||
$duplicates_total = round((float) ($results['duplicates_total'] ?? 0.0), 2);
|
||||
$file_total = round((float) ($results['file_total'] ?? 0.0), 2);
|
||||
|
||||
$summary = [
|
||||
'file_name' => (string) ($results['file_name'] ?? ''),
|
||||
'tag' => (string) ($results['tag'] ?? ''),
|
||||
'created' => (int) ($results['created'] ?? 0),
|
||||
'duplicates' => (int) ($results['duplicates'] ?? 0),
|
||||
'matched' => (int) ($results['matched'] ?? 0),
|
||||
'unmatched' => (int) ($results['unmatched'] ?? 0),
|
||||
'errors' => $results['errors'] ?? [],
|
||||
'duplicates_list' => $results['duplicates_list'] ?? [],
|
||||
'file_total' => $file_total,
|
||||
'created_total' => $created_total,
|
||||
'duplicates_total' => $duplicates_total,
|
||||
// To the centime: every parsed transaction was either created or
|
||||
// recognized as already in base. Any drift means a line vanished
|
||||
// -- never expected, always announced.
|
||||
'totals_ok' => abs($file_total - $created_total - $duplicates_total) < 0.005,
|
||||
];
|
||||
|
||||
\Drupal::service('tempstore.private')->get('figli_compta_ledger')
|
||||
->set('releve_import_result', $summary);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
<?php
|
||||
|
||||
namespace Drupal\figli_compta_ledger\Import;
|
||||
|
||||
/**
|
||||
* One bank statement transaction, in the neutral shape every parser
|
||||
* (CSV today, OFX/CMI maybe later) produces -- the rest of the import
|
||||
* chain (dedup, client matching, node creation) only ever sees this.
|
||||
*
|
||||
* $date: AAAA-MM-JJ (converted from the bank format at parse time).
|
||||
* $montant: signed, negative = money out (Débit), to the centime.
|
||||
* $libelle: raw bank label (full, untruncated in CSV), the basis for
|
||||
* client matching and the line's visible Notes.
|
||||
* $fitid: dedup fingerprint, see CsvReleveParser::fitid().
|
||||
*/
|
||||
final class ReleveTransaction {
|
||||
|
||||
public function __construct(
|
||||
public readonly string $date,
|
||||
public readonly float $montant,
|
||||
public readonly string $libelle,
|
||||
public readonly string $fitid,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Plain-array shape for Batch API serialization (operation args and
|
||||
* sandbox are serialized between requests).
|
||||
*/
|
||||
public function toArray(): array {
|
||||
return [
|
||||
'date' => $this->date,
|
||||
'montant' => $this->montant,
|
||||
'libelle' => $this->libelle,
|
||||
'fitid' => $this->fitid,
|
||||
];
|
||||
}
|
||||
|
||||
public static function fromArray(array $values): self {
|
||||
return new self(
|
||||
(string) $values['date'],
|
||||
(float) $values['montant'],
|
||||
(string) $values['libelle'],
|
||||
(string) $values['fitid'],
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
<?php
|
||||
|
||||
namespace Drupal\figli_compta_ledger;
|
||||
|
||||
/**
|
||||
* Request-scoped opt-out of the répartition invariant check in
|
||||
* figli_compta_ledger_node_presave().
|
||||
*
|
||||
* The historical state key ('figli_compta_ledger.skip_validation') is a flag
|
||||
* shared by every request on the site, stored in the database: while a
|
||||
* programmatic save (inline type change, bank statement import) holds it,
|
||||
* a *concurrent* normal form save happening in another PHP-FPM request would
|
||||
* silently skip validation too -- exactly the kind of hole an integrity
|
||||
* check must never have. This service lives in the dependency injection
|
||||
* container of its own request, so a skip here is physically invisible to
|
||||
* every other request; the depth counter makes nested skips safe and the
|
||||
* try/finally in skip() guarantees it unwinds on exceptions as well as on
|
||||
* normal completion.
|
||||
*
|
||||
* The state key is still honored by node_presave() for backward compatibility
|
||||
* with already-shipped migration scripts, but new code (batch imports,
|
||||
* future migrations) must use this service instead.
|
||||
*/
|
||||
final class SkipValidationContext {
|
||||
|
||||
/**
|
||||
* Current skip depth (0 = validation active).
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
private int $depth = 0;
|
||||
|
||||
/**
|
||||
* Runs $operation with the répartition-sum check disabled for this
|
||||
* request only, restoring it afterwards whatever happens.
|
||||
*
|
||||
* @param callable $operation
|
||||
* Typically fn () => $node->save().
|
||||
*
|
||||
* @return mixed
|
||||
* Whatever $operation returns.
|
||||
*/
|
||||
public function skip(callable $operation): mixed {
|
||||
$this->depth++;
|
||||
try {
|
||||
return $operation();
|
||||
}
|
||||
finally {
|
||||
$this->depth--;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the répartition-sum check is currently disabled for this
|
||||
* request. Read by figli_compta_ledger_node_presave().
|
||||
*
|
||||
* @return bool
|
||||
* TRUE when a skip() is currently in progress.
|
||||
*/
|
||||
public function isSkipped(): bool {
|
||||
return $this->depth > 0;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user