migration is ok again (remains probleme of ram consuming > 4Go for users

This commit is contained in:
Bachir Soussi Chiadmi 2020-11-17 11:23:03 +01:00
parent ee5f880f2c
commit 8db06621ce
57 changed files with 801 additions and 162 deletions

View File

@ -29,6 +29,7 @@
"drupal/commerce_recurring": "^1.x-dev",
"drupal/commerce_variation_cart_form": "^1.x-dev",
"drupal/computed_field": "^2.0@beta",
"drupal/config_devel": "^1.7",
"drupal/config_split": "^1.4",
"drupal/console": "^1.0.2",
"drupal/consumer_image_styles": "^3.0",
@ -44,8 +45,9 @@
"drupal/login_history": "^1.x-dev",
"drupal/mailgun": "^1.x-dev",
"drupal/mailsystem": "^4.1",
"drupal/migrate_plus": "^4.x-dev",
"drupal/migrate_tools": "^4.x-dev",
"drupal/migrate_devel": "2.x-dev@dev",
"drupal/migrate_plus": "5.x-dev@dev",
"drupal/migrate_tools": "5.x-dev@dev",
"drupal/mimemail": "^1.x-dev",
"drupal/modules_weight": "^1.8",
"drupal/permissions_filter": "^1.1",

372
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "056bae3f4b18456168d534660f75a387",
"content-hash": "0fde84a349225b506081d88d67aa5cbc",
"packages": [
{
"name": "alchemy/zippy",
@ -8558,90 +8558,27 @@
"time": "2020-04-25T06:50:43+00:00"
},
{
"name": "drupal/migrate_plus",
"version": "dev-4.x",
"name": "drupal/migrate_devel",
"version": "dev-2.x",
"source": {
"type": "git",
"url": "https://git.drupalcode.org/project/migrate_plus.git",
"reference": "8182985a29e4d839a63b57ec29fb1902d8fad4d7"
"url": "https://git.drupalcode.org/project/migrate_devel.git",
"reference": "c60d049d91389bde0cc5368fabfda24c5eb61780"
},
"require": {
"drupal/core": "^8.3"
"drupal/core": "^8 || ^9"
},
"require-dev": {
"drupal/entity": "*",
"drupal/migrate_example_advanced_setup": "*",
"drupal/migrate_example_setup": "*",
"drupal/migrate_tools": "*",
"drupal/profile": "*"
},
"suggest": {
"ext-soap": "*",
"sainsburys/guzzle-oauth2-plugin": "3.0 required for the OAuth2 authentication plugin"
"conflict": {
"drush/drush": "<9"
},
"type": "drupal-module",
"extra": {
"branch-alias": {
"dev-4.x": "4.x-dev"
"dev-2.x": "2.x-dev"
},
"drupal": {
"version": "8.x-4.2+10-dev",
"datestamp": "1578062586",
"security-coverage": {
"status": "not-covered",
"message": "Dev releases are not covered by Drupal security advisories."
}
}
},
"notification-url": "https://packages.drupal.org/8/downloads",
"license": [
"GPL-2.0+"
],
"authors": [
{
"name": "Mike Ryan",
"homepage": "https://www.drupal.org/u/mikeryan",
"role": "Maintainer"
},
{
"name": "mikeryan",
"homepage": "https://www.drupal.org/user/4420"
}
],
"description": "Enhancements to core migration support.",
"homepage": "https://www.drupal.org/project/migrate_plus",
"support": {
"source": "https://cgit.drupalcode.org/migrate_plus",
"issues": "https://www.drupal.org/project/issues/migrate_plus",
"irc": "irc://irc.freenode.org/drupal-migrate"
},
"time": "2020-01-03T14:41:04+00:00"
},
{
"name": "drupal/migrate_tools",
"version": "dev-4.x",
"source": {
"type": "git",
"url": "https://git.drupalcode.org/project/migrate_tools.git",
"reference": "1852e052bd07f0aec6d9e18aa34310dd40fe7a32"
},
"require": {
"drupal/core": "^8 || ^9",
"drupal/migrate_plus": "^4 || ^5"
},
"require-dev": {
"drupal/migrate_plus": "4.x-dev",
"drupal/migrate_source_csv": "^2.2",
"drush/drush": "^10"
},
"type": "drupal-module",
"extra": {
"branch-alias": {
"dev-4.x": "4.x-dev"
},
"drupal": {
"version": "8.x-4.5+2-dev",
"datestamp": "1591390147",
"version": "8.x-2.0-alpha1+3-dev",
"datestamp": "1592898665",
"security-coverage": {
"status": "not-covered",
"message": "Dev releases are not covered by Drupal security advisories."
@ -8649,7 +8586,7 @@
},
"drush": {
"services": {
"drush.services.yml": "^9 || ^10"
"drush.services.yml": "^9"
}
}
},
@ -8659,22 +8596,290 @@
],
"authors": [
{
"name": "drupalspoons",
"homepage": "https://www.drupal.org/user/3647684"
"name": "Derimagia",
"homepage": "https://www.drupal.org/user/819640"
},
{
"name": "mikeryan",
"homepage": "https://www.drupal.org/user/4420"
"name": "aczietlow",
"homepage": "https://www.drupal.org/user/1616656"
},
{
"name": "andrewmacpherson",
"homepage": "https://www.drupal.org/user/265648"
}
],
"description": "Migrate Development Tools",
"homepage": "https://www.drupal.org/project/migrate_devel",
"support": {
"source": "https://git.drupalcode.org/project/migrate_devel",
"issues": "https://www.drupal.org/project/issues/migrate_devel"
},
"time": "2020-06-23T07:51:09+00:00"
},
{
"name": "drupal/migrate_plus",
"version": "dev-5.x",
"source": {
"type": "git",
"url": "https://git.drupalcode.org/project/migrate_plus.git",
"reference": "6a1b1b6ba7ea8651f84beafe073aa0b18188fe51"
},
"require": {
"drupal/core": "^8.8 || ^9",
"php": ">=7.1"
},
"require-dev": {
"composer/installers": "^1",
"cweagans/composer-patches": "~1.0",
"drupal/core-composer-scaffold": "^7.0",
"drupal/core-dev": "^7.0",
"drupal/core-recommended": "^7.0",
"drupal/migrate_example_advanced_setup": "*",
"drupal/migrate_example_setup": "*",
"drush/drush": "^10",
"mglaman/phpstan-drupal": "^0.12",
"php-parallel-lint/php-parallel-lint": "^1.2",
"phpstan/phpstan-deprecation-rules": "^0.12",
"zaporylie/composer-drupal-optimizations": "^1.0"
},
"suggest": {
"ext-soap": "*",
"sainsburys/guzzle-oauth2-plugin": "3.0 required for the OAuth2 authentication plugin"
},
"type": "drupal-module",
"extra": {
"branch-alias": {
"dev-5.x": "5.x-dev"
},
"drupal": {
"version": "8.x-5.1+11-dev",
"datestamp": "1593462153",
"security-coverage": {
"status": "not-covered",
"message": "Dev releases are not covered by Drupal security advisories."
}
},
"drush": {
"services": {
"drush.services.yml": "^9 || ^10"
}
},
"installer-paths": {
"web/core": [
"type:drupal-core"
],
"web/libraries/{$name}": [
"type:drupal-library"
],
"web/modules/{$name}": [
"type:drupal-module"
],
"web/profiles/{$name}": [
"type:drupal-profile"
],
"web/themes/{$name}": [
"type:drupal-theme"
],
"drush/{$name}": [
"type:drupal-drush"
]
},
"drupal-scaffold": {
"locations": {
"web-root": "web/"
}
}
},
"autoload": {
"classmap": [
".spoons/ScriptHandler.php"
]
},
"notification-url": "https://packages.drupal.org/8/downloads",
"scripts": {
"si": [
"drush si -v --db-url=sqlite://sites/default/files/.sqlite testing"
],
"phpcs": [
"phpcs --runtime-set ignore_warnings_on_exit 1 --runtime-set ignore_errors_on_exit 1 web/modules/custom"
],
"phpcbf": [
"phpcbf --runtime-set ignore_warnings_on_exit 1 --runtime-set ignore_errors_on_exit 1 web/modules/custom"
],
"lint": [
"parallel-lint --exclude web --exclude vendor ."
],
"webserver": [
"cd web && php -S 0.0.0.0:8888 .ht.router.php"
],
"chromedriver": [
"chromedriver --port=9515 --verbose --whitelisted-ips --log-path=/tmp/chromedriver.log --no-sandbox"
],
"unit": [
"phpunit --verbose web/modules/custom"
],
"phpstan": [
"phpstan analyse web/modules/custom"
],
"post-update-cmd": [
"Spoons\\ScriptHandler::createSymlinks"
]
},
"license": [
"GPL-2.0-or-later"
],
"authors": [
{
"name": "Mike Ryan",
"homepage": "https://www.drupal.org/u/mikeryan",
"role": "Maintainer"
},
{
"name": "Lucas Hedding",
"homepage": "https://www.drupal.org/u/heddn",
"role": "Maintainer"
}
],
"description": "Enhancements to core migration support.",
"homepage": "https://www.drupal.org/project/migrate_plus",
"support": {
"source": "https://git.drupalcode.org/project/migrate_plus",
"issues": "https://www.drupal.org/project/issues/migrate_plus",
"slack": "#migrate"
},
"time": "2020-06-29T20:02:43+00:00"
},
{
"name": "drupal/migrate_tools",
"version": "dev-5.x",
"source": {
"type": "git",
"url": "https://git.drupalcode.org/project/migrate_tools.git",
"reference": "ec05dc7f967db2dce370830754e5c28182e446ae"
},
"require": {
"drupal/core": "^8.8 | ^9",
"drupal/migrate_plus": "^5",
"php": ">=7.1"
},
"require-dev": {
"composer/installers": "^1",
"cweagans/composer-patches": "~1.0",
"drupal/core-composer-scaffold": "^7.0",
"drupal/core-dev": "^7.0",
"drupal/core-recommended": "^7.0",
"drupal/migrate_source_csv": "^3.0",
"drush/drush": "^10",
"mglaman/phpstan-drupal": "^0.12",
"php-parallel-lint/php-parallel-lint": "^1.2",
"phpstan/phpstan-deprecation-rules": "^0.12",
"zaporylie/composer-drupal-optimizations": "^1.0"
},
"suggest": {
"drush/drush": "^9 || ^10"
},
"type": "drupal-module",
"extra": {
"branch-alias": {
"dev-5.x": "5.x-dev"
},
"drupal": {
"version": "8.x-5.0+4-dev",
"datestamp": "1594070373",
"security-coverage": {
"status": "not-covered",
"message": "Dev releases are not covered by Drupal security advisories."
}
},
"drush": {
"services": {
"drush.services.yml": "^9 || ^10"
}
},
"installer-paths": {
"web/core": [
"type:drupal-core"
],
"web/libraries/{$name}": [
"type:drupal-library"
],
"web/modules/{$name}": [
"type:drupal-module"
],
"web/profiles/{$name}": [
"type:drupal-profile"
],
"web/themes/{$name}": [
"type:drupal-theme"
],
"drush/{$name}": [
"type:drupal-drush"
]
},
"drupal-scaffold": {
"locations": {
"web-root": "web/"
}
}
},
"autoload": {
"classmap": [
".spoons/ScriptHandler.php"
]
},
"notification-url": "https://packages.drupal.org/8/downloads",
"scripts": {
"si": [
"drush si -v --db-url=sqlite://sites/default/files/.sqlite testing"
],
"phpcs": [
"phpcs --runtime-set ignore_warnings_on_exit 1 --runtime-set ignore_errors_on_exit 1 web/modules/custom"
],
"phpcbf": [
"phpcbf --runtime-set ignore_warnings_on_exit 1 --runtime-set ignore_errors_on_exit 1 web/modules/custom"
],
"lint": [
"parallel-lint --exclude web --exclude vendor ."
],
"webserver": [
"cd web && php -S 0.0.0.0:8888 .ht.router.php"
],
"chromedriver": [
"chromedriver --port=9515 --verbose --whitelisted-ips --log-path=/tmp/chromedriver.log --no-sandbox"
],
"unit": [
"phpunit --verbose web/modules/custom"
],
"phpstan": [
"phpstan analyse web/modules/custom"
],
"post-update-cmd": [
"Spoons\\ScriptHandler::createSymlinks"
]
},
"license": [
"GPL-2.0-or-later"
],
"authors": [
{
"name": "Mike Ryan",
"homepage": "https://www.drupal.org/u/mikeryan",
"role": "Maintainer"
},
{
"name": "Lucas Hedding",
"homepage": "https://www.drupal.org/u/heddn",
"role": "Maintainer"
}
],
"description": "Tools to assist in developing and running migrations.",
"homepage": "http://drupal.org/project/migrate_tools",
"support": {
"source": "http://cgit.drupalcode.org/migrate_tools",
"issues": "http://drupal.org/project/migrate_tools",
"irc": "irc://irc.freenode.org/drupal-migrate"
"source": "https://git.drupalcode.org/project/migrate_tools",
"issues": "https://www.drupal.org/project/issues/migrate_tools",
"slack": "#migrate"
},
"time": "2020-06-05T20:48:48+00:00"
"time": "2020-10-15T21:16:37+00:00"
},
{
"name": "drupal/mimemail",
@ -17016,6 +17221,7 @@
"drupal/jsonrpc": 20,
"drupal/login_history": 20,
"drupal/mailgun": 20,
"drupal/migrate_devel": 20,
"drupal/migrate_plus": 20,
"drupal/migrate_tools": 20,
"drupal/mimemail": 20,

View File

@ -0,0 +1,8 @@
page_alter: false
raw_names: false
error_handlers:
- 0
rebuild_theme: false
devel_dumper: kint
debug_logfile: 'temporary://drupal_debug.txt'
debug_pre: false

View File

@ -114,6 +114,8 @@ module:
menu_block: 0
menu_ui: 0
migrate: 0
migrate_booster: 0
migrate_devel: 0
migrate_drupal: 0
migrate_drupal_multilingual: 0
migrate_plus: 0

View File

@ -0,0 +1,5 @@
commands:
- 'migrate:import'
- 'migrate:rollback'
_core:
default_config_hash: nfYl86Nk2pBTltcOlJifkjhVxt-CeJM5UurON1_PQDA

View File

@ -2,8 +2,6 @@ uuid: dc56e67a-4372-412e-8bba-18c623f22119
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: s3vx8Rz_WiVJ7xdZk9aSaFRCIAvdHcywmrmhCKdAY6U
id: d7_allpublicfiles
class: null
field_plugin_method: null
@ -17,6 +15,7 @@ migration_group: d7_materio
label: 'Public files'
source:
plugin: d7_pubic_file
batch_size: 5000
scheme: public
constants:
source_base_path: ../../../d7.materio.com/public_html

View File

@ -2,8 +2,6 @@ uuid: 9355d3f2-6ba9-4556-b23c-6f16a957a721
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: EMJZXjJ_nG0xuawQaOH-p_ORBAYmBTmygs_QRgD3qJ4
id: d7_node_article
class: null
field_plugin_method: null

View File

@ -2,8 +2,6 @@ uuid: cecfab1b-41c8-42f0-aa36-5277e9c6d792
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: vLmdf7LyOFNcHdarEDF_W3CV4Wp-K6GdgH_dOwuUHTY
id: d7_node_article_i18n
class: null
field_plugin_method: null

View File

@ -2,8 +2,6 @@ uuid: b3aeafeb-b607-403f-835c-5ed27df20521
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: UpJLhcKQg-FLL7eI8MUvKVK9_zsXMOmQTCpIEkRcnKY
id: d7_node_materiau
class: null
field_plugin_method: null
@ -17,6 +15,7 @@ label: 'Node Materiau'
source:
plugin: d7_node_materiau
node_type: materiau
batch_size: 500
high_water_property:
name: changed
alias: 'n'

View File

@ -2,8 +2,6 @@ uuid: d53a192b-b55a-4f92-8d35-47699914de14
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: Qs6SCZjjvh6GVqAsUnNPqcgKFQDrDg-NQ3yeTRQEFLU
id: d7_node_materiau_i18n
class: null
field_plugin_method: null

View File

@ -1,9 +1,10 @@
uuid: 5040a067-947f-4c18-8ee0-64e3f2737167
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: V4b8Ctj8B8jqXWdO1pnAIWyZxa_cdp-tjxU-swY0jvE
dependencies:
module:
- migrate_drupal
- simplenews
id: d7_simplenews_nodes
class: null
field_plugin_method: null
@ -57,7 +58,7 @@ process:
3: workflow_visible
4: workflow_imported
5: workflow_edited
simplenews_issue:
simplenews_issue/target_id:
-
plugin: extract
source: field_simplenews_term
@ -71,6 +72,15 @@ process:
6374: ze_daily_materio_
6274: materio_newsletter
7881: companies
simplenews_issue/sent_count: snn_sent_subscriber_count
simplenews_issue/status: snn_status
simplenews_issue/error_count:
plugin: default_value
default_value: 0
simplenews_issue/subscribers: snn_sent_subscriber_count
simplenews_issue/handler:
plugin: default_value
default_value: simplenews_all
destination:
plugin: 'entity:node'
migration_dependencies:

View File

@ -1,9 +1,10 @@
uuid: 0cee8f1b-5640-4bf7-8b14-c8e67376c8b9
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: VloCz4g4ojd5g00TY-4icqkDJXC2zOYHP3ysyqL4lUQ
dependencies:
module:
- migrate_drupal
- simplenews
id: d7_simplenews_subscribers
class: null
field_plugin_method: null

View File

@ -2,8 +2,6 @@ uuid: 36e781b5-fffe-49eb-a8c5-b6574e8ea464
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: KD8aokrh9C4c7Cnr91IkjxeMimTrOAqjLSWjRACToJ8
id: d7_taxonomy_term_company
class: null
field_plugin_method: null

View File

@ -2,8 +2,6 @@ uuid: 805f2fcd-3109-4f9e-9c25-2f577d38a4c9
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: Oip5TPNIP8RxEmtUffgwo8AuBwxsYGMdECAxEqmnCiA
id: d7_taxonomy_term_showroom
class: null
field_plugin_method: null

View File

@ -2,8 +2,6 @@ uuid: 71bb63f1-e6bf-42dc-aa09-305be7533901
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: ANwnCRB6FgqpsP5cu1RdbPcXBem0E0qSi-7q3RZjmp8
id: d7_taxonomy_term_tags
class: null
field_plugin_method: null

View File

@ -2,8 +2,6 @@ uuid: d2b12c96-7a79-4534-a988-28be5828951b
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: IVIXijd3hu4c6IYCkPsT81h334IVvQ1mRLonwSCegVc
id: d7_taxonomy_term_tags_i18n
class: null
field_plugin_method: null

View File

@ -2,8 +2,6 @@ uuid: dc6ea56a-d1a9-40a3-8fc0-d07a813c5361
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: 55XmBttIA63YE74GI20R9E_synOOR7E6yObM-HVGKSw
id: d7_taxonomy_term_thesaurus
class: null
field_plugin_method: null

View File

@ -2,8 +2,6 @@ uuid: 455074a6-7256-4424-9306-44be4491ca6c
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: dEUkhkROYAKjrQDitdMPb_P8qPFSTRVEypMkdvRHkdg
id: d7_taxonomy_term_thesaurus_i18n
class: null
field_plugin_method: null

View File

@ -6,8 +6,6 @@ dependencies:
module:
- commerce_product
- commerce_license
_core:
default_config_hash: uBcqzDKpO358QBdXpZONLiQ13aH-bbfmWiN4yiwzKIM
id: d7_uc_roles_license
class: null
field_plugin_method: null

View File

@ -7,8 +7,6 @@ dependencies:
- migrate_plus
- migrate_tools
- profile
_core:
default_config_hash: BXfTTGg9wZCH6eEyUC3OcKgCj56WCCRQMcDLYj9yHfo
id: d7_user_profile
class: null
field_plugin_method: null
@ -21,6 +19,7 @@ migration_group: d7_materio
label: 'D7 User Profiles'
source:
plugin: d7_user_profile
batch_size: 1000
high_water_property:
name: changed
alias: p

View File

@ -2,8 +2,6 @@ uuid: 31b658d4-e019-468f-be15-f1704a3bc2f9
langcode: en
status: true
dependencies: { }
_core:
default_config_hash: MSrCVV09c4n2q36chsUNrsaE87aqSrk3FfoG7QBzuxg
id: d7_users
class: null
field_plugin_method: null
@ -16,6 +14,7 @@ migration_group: d7_materio
label: 'User accounts'
source:
plugin: d7_users
batch_size: 1000
process:
name: name
pass: pass

View File

@ -7,8 +7,6 @@ dependencies:
- migrate_plus
- migrate_tools
- materio_migrate
_core:
default_config_hash: tbcV0glxU9BqeXcSUn9ZKXnOPW-C2v8Q-shvIJ2xNG8
id: d7_materio
label: 'D7 Materio'
description: 'Drupal 7 materio content migration to Drupal 8.'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_creation
to_sid: workflow_imported
roles:
anonymous: anonymous
workflow_author: '0'
anonymous: '0'
admin: '0'
root: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_edited
to_sid: workflow_hidden
roles:
anonymous: anonymous
admin: admin
root: root
workflow_author: '0'
anonymous: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_edited
to_sid: workflow_imported
roles:
anonymous: anonymous
root: root
workflow_author: '0'
anonymous: '0'
admin: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_edited
to_sid: workflow_visible
roles:
anonymous: anonymous
admin: admin
root: root
workflow_author: '0'
anonymous: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_hidden
to_sid: workflow_edited
roles:
anonymous: anonymous
workflow_author: '0'
anonymous: '0'
admin: '0'
root: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_hidden
to_sid: workflow_imported
roles:
anonymous: anonymous
root: root
workflow_author: '0'
anonymous: '0'
admin: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_imported
to_sid: workflow_edited
roles:
anonymous: anonymous
admin: admin
root: root
workflow_author: '0'
anonymous: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_imported
to_sid: workflow_hidden
roles:
anonymous: anonymous
admin: admin
root: root
workflow_author: '0'
anonymous: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_imported
to_sid: workflow_visible
roles:
anonymous: anonymous
admin: admin
root: root
workflow_author: '0'
anonymous: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_visible
to_sid: workflow_edited
roles:
anonymous: anonymous
workflow_author: '0'
anonymous: '0'
admin: '0'
root: '0'

View File

@ -11,7 +11,7 @@ module: workflow
from_sid: workflow_visible
to_sid: workflow_imported
roles:
anonymous: anonymous
root: root
workflow_author: '0'
anonymous: '0'
admin: '0'

View File

@ -3,6 +3,59 @@
export PATH=$Path:/bin:/usr/bin:/usr/local/bin
DATE=`date +%d-%m-%y/%H:%M:%S`
# https://www.mediacurrent.com/blog/memory-management-migrations-drupal-8/
# migration_loop(){
# # Better readability with separation.
# echo "========================";
# # Get the output of the drush status.
# drush_output=$(drush ms | grep $1);
#
# # Split output string into an array.
# output=( $drush_output );
#
# # Output the status items.
# for index in "${!output[@]}"
# do
# if [ $index == "0" ]
# then
# echo "Migration: ${output[index]}";
# fi
# if [ $index == "1" ]
# then
# echo "Status: ${output[index]}";
# fi
# if [ $index == "2" ]
# then
# echo "Total: ${output[index]}";
# fi
# if [ $index == "3" ]
# then
# echo "Imported: ${output[index]}";
# fi
# if [ $index == "4" ]
# then
# echo "Remaining: ${output[index]}";
# fi
# done
#
# # Check if all items were imported.
#
# if [ "${output[4]}" == "0" ]
# then
# echo "No items left to import.";
# else
# echo "There are ${output[4]} remaining ${output[0]} items to be imported.";
# echo "Running command: drush mim $1";
# echo "...";
# # Run the migration until it stops.
# drush mim $1;
# # Run the check on this migration again.
# migration_loop $1;
# fi
# }
echo -e "* * * * * * * * * * * * * * * *"
echo -e "* *"
echo -e "* Migration *"
@ -14,59 +67,75 @@ echo -e "* * * * * * * * * * * * * * * *"
echo " "
echo "Migrating Files"
drush mim d7_allpublicfiles --feedback="1000"
# migration_loop d7_allpublicfiles;
sleep 5
echo " "
echo "Migrating Taxonomy"
echo "Migrating thesaurus"
drush mim d7_taxonomy_term_thesaurus --update --feedback="100"
# migration_loop d7_taxonomy_term_thesaurus;
echo " "
drush mim d7_taxonomy_term_thesaurus_i18n --update --feedback="100"
# migration_loop d7_taxonomy_term_thesaurus_i18n;
sleep 5
echo " "
echo "Migrating tags"
drush mim d7_taxonomy_term_tags --update --feedback="100"
# migration_loop d7_taxonomy_term_tags;
echo " "
drush mim d7_taxonomy_term_tags_i18n --update --feedback="100"
# migration_loop d7_taxonomy_term_tags_i18n;
sleep 5
echo " "
echo "Migrating companies"
drush mim d7_taxonomy_term_company --update --feedback="100"
# migration_loop d7_taxonomy_term_company;
sleep 5
echo " "
echo "Migrating showroom"
drush mim d7_taxonomy_term_showroom --update --feedback="100"
# migration_loop d7_taxonomy_term_showroom;
sleep 5
echo " "
echo "Migrating Users"
drush mim d7_users --update --feedback="1000"
# migration_loop d7_users;
echo " "
drush mim d7_user_profile --feedback="1000"
# migration_loop d7_user_profile;
echo " "
drush mim d7_uc_roles_license --update --feedback="200"
# migration_loop d7_uc_roles_license;
sleep 5
echo " "
echo "Migrating Nodes"
echo "Migrating Materials"
drush mim d7_node_materiau --feedback="1000"
# migration_loop d7_node_materiau;
echo " "
drush mim d7_node_materiau_i18n --feedback="1000"
# migration_loop d7_node_materiau_i18n;
sleep 5
echo " "
echo "Migrating breves"
drush mim d7_node_article --feedback="100"
# migration_loop d7_node_article;
echo " "
drush mim d7_node_article_i18n --feedback="100"
# migration_loop d7_node_article_i18n;
sleep 5
echo " "
echo "Migrating simplenews"
drush mim d7_simplenews_nodes --feedback="200"
# migration_loop d7_simplenews_nodes;
drush mim d7_simplenews_subscribers --update --feedback="1000"
# migration_loop d7_simplenews_subscribers;
sleep 5

View File

@ -24,14 +24,19 @@ function computed_field_field_index_compute($entity_type_manager, $entity, $fiel
$nids = $query->execute();
$index = null;
if(is_array($nids)){
$nid = array_pop($nids);
if ($nid) {
$last = entity_load($entity->getEntityTypeId(), array_pop($nids));
// get identifiants fo these entities
$index = $last->get('field_index')->getValue()[0]['value'] + 1;
}else{
$index = '1';
}
return $index;
}
// if(!$index){
// $index = '1';
// }
return $index ? $index : '1';
}

View File

@ -12,6 +12,7 @@ migration_tags:
source:
plugin: d7_pubic_file
batch_size: 5000
scheme: public
constants:
# The tool configuring this migration must set source_base_path. It

View File

@ -10,6 +10,7 @@ migration_tags:
source:
plugin: d7_node_materiau
node_type: materiau
batch_size: 500
high_water_property:
name: changed
alias: n
@ -51,6 +52,10 @@ process:
plugin: default_value
source: workflow
default_value: 2
# -
# plugin: debug
# source: workflow
# label: workflow_debug
-
plugin: static_map
default_value: 2

View File

@ -62,7 +62,8 @@ process:
4: "workflow_imported"
5: "workflow_edited"
simplenews_issue:
# https://www.drupal.org/project/simplenews/issues/3037694
simplenews_issue/target_id:
-
plugin: extract
source: field_simplenews_term
@ -76,8 +77,15 @@ process:
6374: "ze_daily_materio_"
6274: "materio_newsletter"
7881: "companies"
# Sent status
simplenews_issue/sent_count: snn_sent_subscriber_count
simplenews_issue/status: snn_status
simplenews_issue/error_count:
plugin: default_value
default_value: 0
simplenews_issue/subscribers: snn_sent_subscriber_count
simplenews_issue/handler:
plugin: default_value
default_value: simplenews_all
migration_dependencies:
required:

View File

@ -9,6 +9,7 @@ migration_tags:
source:
plugin: d7_user_profile
batch_size: 1000
high_water_property:
name: changed
alias: p

View File

@ -9,6 +9,7 @@ migration_tags:
source:
plugin: d7_users
batch_size: 1000
destination:
plugin: entity:user

View File

@ -13,11 +13,23 @@ dependencies:
# - drupal:field_group_migrate
- telephone_validation
# config_devel:
# install:
# - migrate_plus.migration_group.d7_materio
# - migrate_plus.migration.d7_taxonomy_term_thesaurus
# - migrate_plus.migration.d7_taxonomy_term_tags
# - migrate_plus.migration.d7_taxonomy_term_showroom
# - migrate_plus.migration.d7_taxonomy_term_company
# - migrate_plus.migration.d7_node_entity_translation_materiau
# https://www.liip.ch/en/blog/using-the-new-drupal-8-migration-api-module
config_devel:
install:
- migrate_plus.migration.d7_users
- migrate_plus.migration.d7_allpublicfiles
- migrate_plus.migration.d7_taxonomy_term_company
- migrate_plus.migration.d7_uc_roles_license
- migrate_plus.migration.d7_simplenews_subscribers
- migrate_plus.migration.d7_node_article
- migrate_plus.migration.d7_simplenews_nodes
- migrate_plus.migration.d7_node_materiau
- migrate_plus.migration.d7_node_materiau_i18n
- migrate_plus.migration.d7_node_article_i18n
- migrate_plus.migration.d7_user_profile
- migrate_plus.migration_group.d7_materio
- migrate_plus.migration.d7_taxonomy_term_showroom
- migrate_plus.migration.d7_taxonomy_term_tags_i18n
- migrate_plus.migration.d7_taxonomy_term_tags
- migrate_plus.migration.d7_taxonomy_term_thesaurus_i18n
- migrate_plus.migration.d7_taxonomy_term_thesaurus

View File

@ -178,7 +178,7 @@ class D7NodeMateriau extends FieldableEntity {
drush_print('WARNING: no workflow');
}
$row->setSourceProperty('workflow', $results);
// drush_print('workflow: '.$results);
// record migration errors in field_memo
if(isset($memo)){
@ -186,7 +186,6 @@ class D7NodeMateriau extends FieldableEntity {
$field_memo[0]['value'] .= "\n".$memo;
$row->setSourceProperty('field_memo', $field_memo);
}
return parent::prepareRow($row);
}

View File

@ -84,9 +84,13 @@ class D7SimplenewsNodes extends FieldableEntity {
])
->orderBy('changed');
$query->innerJoin('node', 'n', static::JOIN);
$query->addField('n', 'uid', 'node_uid');
$query->addField('nr', 'uid', 'revision_uid');
$query->innerJoin('node', 'n', static::JOIN);
$query->innerJoin('simplenews_newsletter', 'snn', 'snn.nid = n.nid');
$query->addField('snn', 'status', 'snn_status');
$query->addField('snn', 'sent_subscriber_count', 'snn_sent_subscriber_count');
if (isset($this->configuration['node_type'])) {
$query->condition('n.type', $this->configuration['node_type']);

View File

@ -149,6 +149,8 @@ class D7TaxonomyTermCompany extends FieldableEntity implements ContainerFactoryP
// drush_print('tode_nid: ');
// print_r($tode_nid);
// drush_print(' ');
// TODO: [error] Drupal\Core\Entity\EntityStorageException: Route "entity.workflow_state.canonical" does not exist. in Drupal\Core\Entity\Sql\SqlContentEntityStorage->save() (line 846 of /var/www/html/d8.materio.com/public_html/web/core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorage.php).
if($tode_nid){
// get the tode fields
foreach ($this->getFields('node', 'company') as $field_name => $field) {

View File

@ -127,7 +127,7 @@ class D7UcRolesLicense extends DrupalSqlBase {
'created',
'modified',
]);
$query->orderBy('created', DESC);
$query->orderBy('created', 'DESC');
$orders = $query->execute()->fetchAll();
// print_r($orders);

View File

@ -21,7 +21,7 @@ class D7Users extends FieldableEntity {
public function query() {
return $this->select('users', 'u')
->fields('u')
->condition('u.uid', 0, '>');
->condition('u.uid', 1, '>');
}
/**

View File

@ -0,0 +1,43 @@
Copied from https://www.drupal.org/sandbox/onkeltem/2828817 and added the patch from https://www.drupal.org/project/2828817/issues/2919993
# About
On Drupal 7 we could disable hooks while running migrations:
https://www.drupal.org/node/2136601
This module adds a similar feature. You can disable hooks
in settings.php or by editing configuratoin object `migrate_booster.settings`.
There are two ways to disable hooks:
1) Disable specific hooks and modules:
```
$config['migrate_booster.settings']['hooks'] = [
# Entity insert
'entity_insert' => [
'workbench_moderation',
'pathauto',
'xmlsitemap',
],
# Entity presave
'entity_presave' => [
'xmlsitemap',
],
# Entity predelete
'entity_predelete' => [
'flag',
],
];
```
2) Disable all hooks of specific modules:
```
$config['migrate_booster.settings']['modules'] = [
'workbench_moderation',
'pathauto',
'xmlsitemap',
];
```

View File

@ -0,0 +1,3 @@
commands:
- migrate:import
- migrate:rollback

View File

@ -0,0 +1,6 @@
services:
migrate_booster.commands:
class: \Drupal\migrate_booster\Commands\MigrateBoosterCommands
arguments: ['@plugin.manager.migration', '@date.formatter', '@entity_type.manager', '@keyvalue']
tags:
- { name: drush.command }

View File

@ -0,0 +1,30 @@
<?php
use Drupal\migrate_booster\MigrateBooster;
/**
* Implements hook_drush_init().
*/
function migrate_booster_drush_init() {
// Skip when bootstrap level is low (e.g. drush cr)
if (!\Drupal::hasContainer()) {
return;
}
MigrateBooster::bootDrush();
}
/**
* Implements hook_drush_command().
*/
function migrate_booster_drush_command() {
$items['migrate-booster-reset'] = [
'description' => 'Resets migrate booster and implementation cache.',
'aliases' => ['mbr'],
'drupal dependencies' => ['migrate_booster'],
];
return $items;
}
function drush_migrate_booster_reset() {
MigrateBooster::disable();
}

View File

@ -0,0 +1,7 @@
name: Migrate booster
type: module
description: Hooks disabler
core: 8.x
package: Migration
dependencies:
- migrate

View File

@ -0,0 +1,15 @@
<?php
use Drupal\migrate_booster\MigrateBooster;
/** @noinspection PhpInconsistentReturnPointsInspection */
/**
* Implements hook_module_implements_alter().
* @param $implementations
* @param $hook
* @return null
*/
function migrate_booster_module_implements_alter(&$implementations, $hook) {
MigrateBooster::alter($implementations, $hook);
}

View File

@ -0,0 +1,5 @@
services:
migrate_booster.hooks_enabler_subscriber:
class: Drupal\migrate_booster\HooksEnablerSubscriber
tags:
- { name: 'event_subscriber' }

View File

@ -0,0 +1,10 @@
<?php
use Drupal\migrate_booster\MigrateBooster;
use Symfony\Component\Console\Event\ConsoleCommandEvent;
use Symfony\Component\Console\ConsoleEvents;
/** @noinspection PhpUnusedParameterInspection */
$GLOBALS['dispatcher']->addListener(ConsoleEvents::COMMAND, function (ConsoleCommandEvent $event) {
MigrateBooster::bootDrupal();
});

View File

@ -0,0 +1,61 @@
<?php
namespace Drupal\migrate_booster\Commands;
use Consolidation\AnnotatedCommand\AnnotationData;
use Drupal\migrate_booster\MigrateBooster;
use Drush\Commands\DrushCommands;
use Symfony\Component\Console\Input\InputInterface;
/**
*
* In addition to a commandfile like this one, you need a drush.services.yml
* in root of your module.
*
* See these files for an example of injecting Drupal services:
* - http://cgit.drupalcode.org/devel/tree/src/Commands/DevelCommands.php
* - http://cgit.drupalcode.org/devel/tree/drush.services.yml
*/
class MigrateBoosterCommands extends DrushCommands {
/**
* Resets migrate booster and implementation cache.
*
* @command migrate:booster:reset
*
* @validate-module-enabled migrate_booster
* @aliases mbr,migrate-booster-reset
*/
public function boosterReset()
{
// See bottom of https://weitzman.github.io/blog/port-to-drush9 for details on what to change when porting a
// legacy command.
MigrateBooster::reset();
}
/**
* Enables migrate booster and implementation cache.
*
* @command migrate:booster:enable
*
* @validate-module-enabled migrate_booster
* @aliases mbe,migrate-booster-enable
*/
public function boosterEnable()
{
// See bottom of https://weitzman.github.io/blog/port-to-drush9 for details on what to change when porting a
// legacy command.
MigrateBooster::enable();
}
/**
* @hook init *
*/
public function initCommand(InputInterface $input, AnnotationData $annotationData) {
// Skip when bootstrap level is low (e.g. drush cr)
if (!\Drupal::hasContainer()) {
return;
}
MigrateBooster::bootDrush($input, $annotationData);
}
}

View File

@ -0,0 +1,29 @@
<?php
namespace Drupal\migrate_booster;
use Symfony\Component\HttpKernel\KernelEvents;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
/**
* Provides a MyModuleSubscriber.
*/
class HooksEnablerSubscriber implements EventSubscriberInterface {
/**
* {@inheritdoc}
*/
static function getSubscribedEvents() {
$events[KernelEvents::REQUEST][] = array('ensureHooksEnabled', 20);
return $events;
}
/**
* Triggers on 'kernel.request' event which occurs when Drupal
* bootstraps (but not when Drush or Drupal console command runs).
*/
public function ensureHooksEnabled() {
MigrateBooster::bootDrupal();
}
}

View File

@ -0,0 +1,115 @@
<?php
namespace Drupal\migrate_booster;
use Consolidation\AnnotatedCommand\AnnotationData;
use Symfony\Component\Console\Input\InputInterface;
class MigrateBooster {
protected static $alterActive;
protected static $config;
const CID = 'migrate_booster_enabled';
// Startup hooks
/**
* Reacts on HOOK_drush_init().
*
* Enables/disables booster depending on a drush command invoked.
*/
public static function bootDrush(InputInterface $input, AnnotationData $annotationData) {
if (in_array($annotationData['command'], static::getConfig('commands'))) {
static::enable();
}
else {
static::disable();
}
}
/**
* Disables booster on Drupal and Drupal console boots.
*/
public static function bootDrupal() {
static::disable();
}
/**
* Enables booster.
*
* Resets implementation cache and sets $alterActive class variable.
*
*/
public static function enable() {
static::$alterActive = TRUE;
static::reset();
}
/**
* Disables booster.
*
* Resets implementation cache.
*/
public static function disable() {
static::reset();
}
/**
* Resets implementations cache.
*/
public static function reset() {
$module_handler = \Drupal::moduleHandler();
$module_handler->resetImplementations();
}
/** @noinspection PhpInconsistentReturnPointsInspection */
/**
* Implements hook_module_implementation_alter().
*
* Disables configured hooks.
* @param $implementations
* @param $hook
* @return null
*/
public static function alter(&$implementations, $hook) {
if (!static::$alterActive) {
return NULL;
}
if (!$implementations) {
return NULL;
}
$hooks = static::getConfig('hooks');
$modules = static::getConfig('modules');
$disabled = [];
// Disable by hook + module
if (array_key_exists($hook, $hooks)) {
$disabled = array_intersect_key($implementations, array_flip($hooks[$hook]));
}
// Disable by module
$disabled += array_intersect_key($implementations, array_flip($modules));
$implementations = array_diff_key($implementations, $disabled);
/** @noinspection PhpUnusedParameterInspection */
array_walk($disabled, function ($el, $key) use ($hook) {
error_log('DISABLED: ' . $key . '_' . $hook);
});
}
/**
* Helper functions
*
* @param $key
* @return array
*/
protected static function getConfig($key) {
if (!static::$config) {
static::$config = \Drupal::config('migrate_booster.settings')->get();
}
if ($key && isset(static::$config[$key])) {
return static::$config[$key];
}
return [];
}
}