migrate.module 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603
  1. <?php
  2. /**
  3. * @file
  4. * API and drush commands to support migration of data from external sources
  5. * into a Drupal installation.
  6. */
  7. // TODO:
  8. // Continue hook_schema_alter() for map & message tables?
  9. // Views hooks for map/message tables
  10. // xlat support?
  11. // Documentation
  12. // Tests
  13. define('MIGRATE_API_VERSION', 2);
  14. define('MIGRATE_ACCESS_BASIC', 'migration information');
  15. define('MIGRATE_ACCESS_ADVANCED', 'advanced migration information');
  16. /**
  17. * Implements hook_help().
  18. */
  19. function migrate_help($path, $arg) {
  20. switch ($path) {
  21. case 'admin/help#migrate':
  22. $output = '';
  23. $output .= '<h3>' . t('About') . '</h3>';
  24. $output .= '<p>' . t('The Migrate module provides a flexible framework for migrating content into Drupal from other sources (e.g., when converting a web site from another CMS to Drupal). Out-of-the-box, support for creating Drupal nodes, taxonomy terms, comments, and users are included. Plugins permit migration of other types of content. For more information, see the online documentation for the <a href="@handbook">Migrate module</a>.', array('@handbook' => 'http://drupal.org/migrate')) . '</p>';
  25. $output .= '<h3>' . t('Uses') . '</h3>';
  26. $output .= '<dl>';
  27. $output .= '<dt>' . t('Getting Started') . '</dt>';
  28. $output .= '<dd>' . t('To get started, enable the migrate_example module and browse to admin/content/migrate to see its dashboard. The code for this migration is in migrate_example/beer.inc (advanced examples are in wine.inc). Mimic that file in order to specify your own migrations.') . '</dd>';
  29. $output .= '<dt>' . t('Migrate Support') . '</dt>';
  30. $output .= '<dd>' . t('The Migrate module itself has support for migration into core objects. Some built-in support for migration involving contrib modules is in the migrate_extras module.') . '</dd>';
  31. $output .= '<dt>' . t('Migrate Extended Support') . '</dt>';
  32. $output .= '<dd>' . t('The best place to implement migration support for a contributed module is in that module, not in the Migrate or Migrate Extras modules. That way, the migration support is always self-consistent with the current module implementation - it is not practical for the migrate modules to keep up with changes to all other contrib modules.') . '</dd>';
  33. $output .= '</dl>';
  34. return $output;
  35. }
  36. }
  37. /**
  38. * Retrieve a list of all active migrations, ordered by dependencies. To be
  39. * recognized, a class must be non-abstract, and derived from MigrationBase.
  40. *
  41. * @param $reset
  42. * If TRUE, the static cache of migrations will be flushed before attempting to
  43. * reinstantiate all active migrations. This can be important for script runs
  44. * where migration classes may be dynamically registered.
  45. *
  46. * @return
  47. * Array of migration objects, keyed by the machine name.
  48. */
  49. function migrate_migrations($reset = NULL) {
  50. static $migrations = array();
  51. if (!empty($migrations) && empty($reset)) {
  52. return $migrations;
  53. }
  54. // Get list of modules implementing Migrate API - mainly, we're looking to
  55. // make sure any dynamic migrations defined in hook_migrate_api() get registered.
  56. migrate_get_module_apis(TRUE);
  57. $dependencies_list = array();
  58. $dependent_migrations = array();
  59. $required_migrations = array();
  60. $result = db_select('migrate_status', 'ms')
  61. ->fields('ms', array('machine_name', 'class_name'))
  62. ->execute();
  63. foreach ($result as $row) {
  64. if (class_exists($row->class_name)) {
  65. $reflect = new ReflectionClass($row->class_name);
  66. if (!$reflect->isAbstract() && $reflect->isSubclassOf('MigrationBase')) {
  67. $migration = MigrationBase::getInstance($row->machine_name);
  68. if ($migration) {
  69. $dependencies = $migration->getDependencies();
  70. $dependencies_list[$row->machine_name] = $dependencies;
  71. if (count($dependencies) > 0) {
  72. // Set classes with dependencies aside for reordering
  73. $dependent_migrations[$row->machine_name] = $migration;
  74. $required_migrations += $dependencies;
  75. }
  76. else {
  77. // No dependencies, just add
  78. $migrations[$row->machine_name] = $migration;
  79. }
  80. }
  81. }
  82. else {
  83. MigrationBase::displayMessage(t('Class !class is no longer a valid concrete migration class',
  84. array('!class' => $row->class_name)));
  85. }
  86. }
  87. else {
  88. MigrationBase::displayMessage(t('Class !class could not be found',
  89. array('!class' => $row->class_name)));
  90. }
  91. }
  92. $ordered_migrations = migrate_order_dependencies($dependencies_list);
  93. foreach ($ordered_migrations as $name) {
  94. if (!isset($migrations[$name])) {
  95. $migrations[$name] = $dependent_migrations[$name];
  96. }
  97. }
  98. // The migrations are now ordered according to their own dependencies - now order
  99. // them by group
  100. $groups = MigrateGroup::groups();
  101. // Seed the final list by properly-ordered groups.
  102. $final_migrations = array();
  103. foreach ($groups as $name => $group) {
  104. $final_migrations[$name] = array();
  105. }
  106. // Fill in the grouped list.
  107. foreach ($migrations as $machine_name => $migration) {
  108. if (!method_exists($migration, 'getGroup')) {
  109. MigrationBase::displayMessage(t('Migration !machine_name is not a valid Migration dependency.', array(
  110. '!machine_name' => $machine_name,
  111. )));
  112. }
  113. else {
  114. $final_migrations[$migration->getGroup()
  115. ->getName()][$machine_name] = $migration;
  116. }
  117. }
  118. // Flatten the grouped list.
  119. $migrations = array();
  120. foreach ($final_migrations as $group_name => $group_migrations) {
  121. foreach ($group_migrations as $machine_name => $migration) {
  122. $migrations[$machine_name] = $migration;
  123. }
  124. }
  125. return $migrations;
  126. }
  127. /**
  128. * Invoke any available handlers attached to a given destination type.
  129. * If any handlers have dependencies defined, they will be invoked after
  130. * the specified handlers.
  131. *
  132. * @param $destination
  133. * Destination type ('Node', 'User', etc.) - generally the same string as
  134. * the destination class name without the MigrateDestination prefix.
  135. * @param $method
  136. * Method name such as 'prepare' (called at the beginning of an import
  137. * operation) or 'complete' (called at the end of an import operation).
  138. * @param ...
  139. * Parameters to be passed to the handler.
  140. */
  141. function migrate_handler_invoke_all($destination, $method) {
  142. $args = func_get_args();
  143. array_shift($args);
  144. array_shift($args);
  145. $return = array();
  146. $class_list = _migrate_class_list('MigrateDestinationHandler');
  147. $disabled = unserialize(variable_get('migrate_disabled_handlers', serialize(array())));
  148. foreach ($class_list as $class_name => $handler) {
  149. if (!in_array($class_name, $disabled) && $handler->handlesType($destination)
  150. && method_exists($handler, $method)) {
  151. migrate_instrument_start($class_name . '->' . $method);
  152. $result = call_user_func_array(array($handler, $method), $args);
  153. migrate_instrument_stop($class_name . '->' . $method);
  154. if (isset($result) && is_array($result)) {
  155. $return = array_merge($return, $result);
  156. }
  157. elseif (isset($result)) {
  158. $return[] = $result;
  159. }
  160. }
  161. }
  162. return $return;
  163. }
  164. /**
  165. * Invoke any available handlers attached to a given field type.
  166. *
  167. * @param $entity
  168. * The object we are building up before calling example_save().
  169. * @param $field_info
  170. * Array of info on the field, from field_info_field().
  171. * @param $instance
  172. * Array of info in the field instance, from field_info_instances().
  173. * @param $values
  174. * Array of incoming values, to be transformed into the appropriate structure
  175. * for the field type.
  176. * @param $method
  177. * Handler method to call (defaults to prepare()).
  178. */
  179. function migrate_field_handler_invoke_all($entity, array $field_info,
  180. array $instance, array $values, $method = 'prepare') {
  181. $return = array();
  182. $type = $field_info['type'];
  183. $class_list = _migrate_class_list('MigrateFieldHandler');
  184. $disabled = unserialize(variable_get('migrate_disabled_handlers',
  185. serialize(array())));
  186. $handler_called = FALSE;
  187. foreach ($class_list as $class_name => $handler) {
  188. if (!in_array($class_name, $disabled) && $handler->handlesType($type)
  189. && method_exists($handler, $method)) {
  190. migrate_instrument_start($class_name . '->' . $method);
  191. $result = call_user_func_array(array($handler, $method),
  192. array($entity, $field_info, $instance, $values));
  193. $handler_called = TRUE;
  194. migrate_instrument_stop($class_name . '->' . $method);
  195. if (isset($result) && is_array($result)) {
  196. $return = array_merge_recursive($return, $result);
  197. }
  198. elseif (isset($result)) {
  199. $return[] = $result;
  200. }
  201. }
  202. }
  203. if (!$handler_called && $method == 'prepare') {
  204. $handler = new MigrateDefaultFieldHandler();
  205. migrate_instrument_start('MigrateDefaultFieldHandler->prepare');
  206. $result = call_user_func_array(array($handler, 'prepare'),
  207. array($entity, $field_info, $instance, $values));
  208. migrate_instrument_stop('MigrateDefaultFieldHandler->prepare');
  209. if (isset($result) && is_array($result)) {
  210. $return = array_merge_recursive($return, $result);
  211. }
  212. elseif (isset($result)) {
  213. $return[] = $result;
  214. }
  215. }
  216. return $return;
  217. }
  218. /**
  219. * For a given parent class, identify and instantiate objects for any
  220. * non-abstract classes derived from the parent, returning an array of the
  221. * objects indexed by class name. The array will be ordered such that any
  222. * classes with dependencies are listed after the classes they are dependent
  223. * on.
  224. *
  225. * @param $parent_class
  226. * Name of a class from which results will be derived.
  227. *
  228. * @return
  229. * Array of objects, keyed by the class name.
  230. */
  231. function _migrate_class_list($parent_class) {
  232. // Get info on modules implementing Migrate API
  233. static $module_info;
  234. if (!isset($module_info)) {
  235. $module_info = migrate_get_module_apis();
  236. }
  237. static $class_lists = array();
  238. if (!isset($class_lists[$parent_class])) {
  239. $class_lists[$parent_class] = array();
  240. if ($parent_class == 'MigrateDestinationHandler') {
  241. $handler_key = 'destination handlers';
  242. }
  243. else {
  244. $handler_key = 'field handlers';
  245. }
  246. // Add explicitly-registered handler classes
  247. foreach ($module_info as $info) {
  248. if (isset($info[$handler_key]) && is_array($info[$handler_key])) {
  249. foreach ($info[$handler_key] as $handler_class) {
  250. $class_lists[$parent_class][$handler_class] = new $handler_class();
  251. }
  252. }
  253. }
  254. }
  255. return $class_lists[$parent_class];
  256. }
  257. /**
  258. * Implements hook_hook_info().
  259. */
  260. function migrate_hook_info() {
  261. // Look for hook_migrate_api() in example.migrate.inc.
  262. $hooks['migrate_api'] = array(
  263. 'group' => 'migrate',
  264. );
  265. $hooks['migrate_api_alter'] = array(
  266. 'group' => 'migrate',
  267. );
  268. return $hooks;
  269. }
  270. /**
  271. * Implementation of hook_permission().
  272. */
  273. function migrate_permission() {
  274. return array(
  275. MIGRATE_ACCESS_BASIC => array(
  276. 'title' => t('Access to basic migration information'),
  277. ),
  278. MIGRATE_ACCESS_ADVANCED => array(
  279. 'title' => t('Access to advanced migration information'),
  280. ),
  281. );
  282. }
  283. /**
  284. * Get a list of modules that support the current migrate API.
  285. */
  286. function migrate_get_module_apis($reset = FALSE) {
  287. static $cache = NULL;
  288. if ($reset) {
  289. $cache = NULL;
  290. }
  291. if (!isset($cache)) {
  292. $cache = array();
  293. foreach (module_implements('migrate_api') as $module) {
  294. $function = $module . '_migrate_api';
  295. $info = $function();
  296. if (isset($info['api']) && $info['api'] == MIGRATE_API_VERSION) {
  297. $cache[$module] = $info;
  298. }
  299. else {
  300. drupal_set_message(t('%function supports Migrate API version %modversion,
  301. Migrate module API version is %version - migration support not loaded.',
  302. array(
  303. '%function' => $function,
  304. '%modversion' => $info['api'],
  305. '%version' => MIGRATE_API_VERSION,
  306. )));
  307. }
  308. }
  309. // Allow modules to alter the migration information.
  310. drupal_alter('migrate_api', $cache);
  311. }
  312. return $cache;
  313. }
  314. /**
  315. * Register any migrations defined in hook_migrate_api().
  316. *
  317. * @param array $machine_names
  318. * If populated, only (re)register the specified migrations.
  319. */
  320. function migrate_static_registration($machine_names = array()) {
  321. $module_info = migrate_get_module_apis(TRUE);
  322. foreach ($module_info as $module => $info) {
  323. // Register any groups defined via the hook.
  324. if (isset($info['groups']) && is_array($info['groups'])) {
  325. foreach ($info['groups'] as $name => $arguments) {
  326. $title = $arguments['title'];
  327. unset($arguments['title']);
  328. MigrateGroup::register($name, $title, $arguments);
  329. }
  330. }
  331. // Register any migrations defined via the hook.
  332. if (isset($info['migrations']) && is_array($info['migrations'])) {
  333. foreach ($info['migrations'] as $machine_name => $arguments) {
  334. // If we have an explicit list to register, skip any not in the list.
  335. if (!empty($machine_names) && !in_array($machine_name, $machine_names)) {
  336. continue;
  337. }
  338. $class_name = $arguments['class_name'];
  339. unset($arguments['class_name']);
  340. // Call the right registerMigration implementation. Note that this means
  341. // that classes that override registerMigration() must always call it
  342. // directly, they cannot register those classes by defining them in
  343. // hook_migrate_api() and expect their extension to be called.
  344. if (is_subclass_of($class_name, 'Migration')) {
  345. Migration::registerMigration($class_name, $machine_name, $arguments);
  346. }
  347. else {
  348. MigrationBase::registerMigration($class_name, $machine_name, $arguments);
  349. }
  350. }
  351. }
  352. }
  353. }
  354. /**
  355. * Do a topological sort on our dependencies graph.
  356. */
  357. function migrate_order_dependencies($dependencies) {
  358. $visited = array();
  359. $list = array();
  360. foreach (array_keys($dependencies) as $name) {
  361. $visited[$name] = FALSE;
  362. }
  363. foreach (array_keys($dependencies) as $name) {
  364. migrate_visit_dependent($dependencies, $name, $list, $visited);
  365. }
  366. return $list;
  367. }
  368. /**
  369. * Depth-first search for independent migrations.
  370. */
  371. function migrate_visit_dependent($dependencies, $name, &$list, &$visited) {
  372. if ($visited[$name]) {
  373. if ($list[$name]) {
  374. return;
  375. }
  376. else {
  377. throw new MigrateException(t('Failure to sort migration list due to circular dependencies involving %name.', array('%name' => $name)));
  378. }
  379. }
  380. $visited[$name] = TRUE;
  381. if (isset($dependencies[$name])) {
  382. foreach ($dependencies[$name] as $dependent) {
  383. migrate_visit_dependent($dependencies, $dependent, $list, $visited);
  384. }
  385. }
  386. $list[$name] = $name;
  387. }
  388. /**
  389. * Implements hook_watchdog().
  390. * Find the migration that is currently running and notify it.
  391. *
  392. * @param array $log_entry
  393. */
  394. function migrate_watchdog($log_entry) {
  395. // Ensure that the Migration class exists, as different bootstrap phases may
  396. // not have included migration.inc yet.
  397. if (class_exists('Migration') && $migration = Migration::currentMigration()) {
  398. switch ($log_entry['severity']) {
  399. case WATCHDOG_EMERGENCY:
  400. case WATCHDOG_ALERT:
  401. case WATCHDOG_CRITICAL:
  402. case WATCHDOG_ERROR:
  403. $severity = MigrationBase::MESSAGE_ERROR;
  404. break;
  405. case WATCHDOG_WARNING:
  406. $severity = MigrationBase::MESSAGE_WARNING;
  407. break;
  408. case WATCHDOG_NOTICE:
  409. $severity = MigrationBase::MESSAGE_NOTICE;
  410. break;
  411. case WATCHDOG_DEBUG:
  412. case WATCHDOG_INFO:
  413. default:
  414. $severity = MigrationBase::MESSAGE_INFORMATIONAL;
  415. break;
  416. }
  417. $variables = is_array($log_entry['variables']) ? $log_entry['variables'] : array();
  418. $migration->saveMessage(t($log_entry['message'], $variables), $severity);
  419. }
  420. }
  421. /**
  422. * Resource functions modeled on Drupal's timer functions
  423. */
  424. /**
  425. * Save memory usage with the specified name. If you start and stop the same
  426. * memory name multiple times, the measured differences will be accumulated.
  427. *
  428. * @param name
  429. * The name of the memory measurement.
  430. */
  431. function migrate_memory_start($name) {
  432. global $_migrate_memory;
  433. $_migrate_memory[$name]['start'] = memory_get_usage();
  434. $_migrate_memory[$name]['count'] =
  435. isset($_migrate_memory[$name]['count']) ? ++$_migrate_memory[$name]['count'] : 1;
  436. }
  437. /**
  438. * Read the current memory value without recording the change.
  439. *
  440. * @param name
  441. * The name of the memory measurement.
  442. *
  443. * @return
  444. * The change in bytes since the last start.
  445. */
  446. function migrate_memory_read($name) {
  447. global $_migrate_memory;
  448. if (isset($_migrate_memory[$name]['start'])) {
  449. $stop = memory_get_usage();
  450. $diff = $stop - $_migrate_memory[$name]['start'];
  451. if (isset($_migrate_memory[$name]['bytes'])) {
  452. $diff += $_migrate_memory[$name]['bytes'];
  453. }
  454. return $diff;
  455. }
  456. return $_migrate_memory[$name]['bytes'];
  457. }
  458. /**
  459. * Stop the memory counter with the specified name.
  460. *
  461. * @param name
  462. * The name of the memory measurement.
  463. *
  464. * @return
  465. * A memory array. The array contains the number of times the memory has been
  466. * started and stopped (count) and the accumulated memory difference value in
  467. * bytes.
  468. */
  469. function migrate_memory_stop($name) {
  470. global $_migrate_memory;
  471. if (isset($_migrate_memory[$name])) {
  472. if (isset($_migrate_memory[$name]['start'])) {
  473. $stop = memory_get_usage();
  474. $diff = $stop - $_migrate_memory[$name]['start'];
  475. if (isset($_migrate_memory[$name]['bytes'])) {
  476. $_migrate_memory[$name]['bytes'] += $diff;
  477. }
  478. else {
  479. $_migrate_memory[$name]['bytes'] = $diff;
  480. }
  481. unset($_migrate_memory[$name]['start']);
  482. }
  483. return $_migrate_memory[$name];
  484. }
  485. }
  486. /**
  487. * Start measuring time and (optionally) memory consumption over a section of
  488. * code. Note that the memory consumption measurement is generally not useful
  489. * in lower areas of the code, where data is being generated that will be freed
  490. * by the next call to the same area. For example, measuring the memory
  491. * consumption of db_query is not going to be helpful.
  492. *
  493. * @param $name
  494. * The name of the measurement.
  495. * @param $include_memory
  496. * Measure both memory and timers. Defaults to FALSE (timers only).
  497. */
  498. function migrate_instrument_start($name, $include_memory = FALSE) {
  499. global $_migrate_track_memory, $_migrate_track_timer;
  500. if ($_migrate_track_memory && $include_memory) {
  501. migrate_memory_start($name);
  502. }
  503. if ($_migrate_track_timer) {
  504. timer_start($name);
  505. }
  506. }
  507. /**
  508. * Stop measuring both memory and time consumption over a section of code.
  509. *
  510. * @param $name
  511. * The name of the measurement.
  512. */
  513. function migrate_instrument_stop($name) {
  514. global $_migrate_track_memory, $_migrate_track_timer;
  515. if ($_migrate_track_timer) {
  516. timer_stop($name);
  517. }
  518. if ($_migrate_track_memory) {
  519. migrate_memory_stop($name);
  520. }
  521. }
  522. /**
  523. * Call hook_migrate_overview for overall documentation on implemented
  524. * migrations.
  525. */
  526. function migrate_overview() {
  527. $overview = '';
  528. $results = module_invoke_all('migrate_overview');
  529. foreach ($results as $result) {
  530. $overview .= $result . ' ';
  531. }
  532. return $overview;
  533. }
  534. /**
  535. * Implements hook_modules_enabled.
  536. */
  537. function migrate_modules_enabled($modules) {
  538. if (array_intersect($modules, module_implements('migrate_api'))) {
  539. migrate_static_registration();
  540. }
  541. }
  542. /**
  543. * Implements hook_module_implements_alter().
  544. */
  545. function migrate_module_implements_alter(&$implementation, $hook) {
  546. // Ensure that the Migration class exists, as different bootstrap phases may
  547. // not have included migration.inc yet.
  548. if (class_exists('Migration') && $migration = Migration::currentMigration()) {
  549. $disable_hooks = $migration->getDisableHooks();
  550. if (isset($disable_hooks[$hook])) {
  551. foreach ($disable_hooks[$hook] as $module) {
  552. unset($implementation[$module]);
  553. }
  554. }
  555. }
  556. }