migrate.module 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564
  1. <?php
  2. /**
  3. * @file
  4. * API and drush commands to support migration of data from external sources
  5. * into a Drupal installation.
  6. */
  7. // TODO:
  8. // Continue hook_schema_alter() for map & message tables?
  9. // Views hooks for map/message tables
  10. // xlat support?
  11. // Documentation
  12. // Tests
  13. define('MIGRATE_API_VERSION', 2);
  14. define('MIGRATE_ACCESS_BASIC', 'migration information');
  15. define('MIGRATE_ACCESS_ADVANCED', 'advanced migration information');
  16. /**
  17. * Retrieve a list of all active migrations, ordered by dependencies. To be
  18. * recognized, a class must be non-abstract, and derived from MigrationBase.
  19. *
  20. * @param $reset
  21. * If TRUE, the static cache of migrations will be flushed before attempting to
  22. * reinstantiate all active migrations. This can be important for script runs
  23. * where migration classes may be dynamically registered.
  24. *
  25. * @return
  26. * Array of migration objects, keyed by the machine name.
  27. */
  28. function migrate_migrations($reset = NULL) {
  29. static $migrations = array();
  30. if (!empty($migrations) && empty($reset)) {
  31. return $migrations;
  32. }
  33. // Get list of modules implementing Migrate API - mainly, we're looking to
  34. // make sure any dynamic migrations defined in hook_migrate_api() get registered.
  35. migrate_get_module_apis(TRUE);
  36. $dependencies_list = array();
  37. $dependent_migrations = array();
  38. $required_migrations = array();
  39. $result = db_select('migrate_status', 'ms')
  40. ->fields('ms', array('machine_name', 'class_name'))
  41. ->execute();
  42. foreach ($result as $row) {
  43. if (class_exists($row->class_name)) {
  44. $reflect = new ReflectionClass($row->class_name);
  45. if (!$reflect->isAbstract() && $reflect->isSubclassOf('MigrationBase')) {
  46. $migration = MigrationBase::getInstance($row->machine_name);
  47. if ($migration) {
  48. $dependencies = $migration->getDependencies();
  49. $dependencies_list[$row->machine_name] = $dependencies;
  50. if (count($dependencies) > 0) {
  51. // Set classes with dependencies aside for reordering
  52. $dependent_migrations[$row->machine_name] = $migration;
  53. $required_migrations += $dependencies;
  54. }
  55. else {
  56. // No dependencies, just add
  57. $migrations[$row->machine_name] = $migration;
  58. }
  59. }
  60. }
  61. else {
  62. MigrationBase::displayMessage(t('Class !class is no longer a valid concrete migration class',
  63. array('!class' => $row->class_name)));
  64. }
  65. }
  66. else {
  67. MigrationBase::displayMessage(t('Class !class no longer exists',
  68. array('!class' => $row->class_name)));
  69. }
  70. }
  71. $ordered_migrations = migrate_order_dependencies($dependencies_list);
  72. foreach ($ordered_migrations as $name) {
  73. if (!isset($migrations[$name])) {
  74. $migrations[$name] = $dependent_migrations[$name];
  75. }
  76. }
  77. // The migrations are now ordered according to their own dependencies - now order
  78. // them by group
  79. $groups = MigrateGroup::groups();
  80. // Seed the final list by properly-ordered groups.
  81. $final_migrations = array();
  82. foreach ($groups as $name => $group) {
  83. $final_migrations[$name] = array();
  84. }
  85. // Fill in the grouped list
  86. foreach ($migrations as $machine_name => $migration) {
  87. $final_migrations[$migration->getGroup()->getName()][$machine_name] = $migration;
  88. }
  89. // Then flatten the list
  90. $migrations = array();
  91. foreach ($final_migrations as $group_name => $group_migrations) {
  92. foreach ($group_migrations as $machine_name => $migration) {
  93. $migrations[$machine_name] = $migration;
  94. }
  95. }
  96. return $migrations;
  97. }
  98. /**
  99. * Invoke any available handlers attached to a given destination type.
  100. * If any handlers have dependencies defined, they will be invoked after
  101. * the specified handlers.
  102. *
  103. * @param $destination
  104. * Destination type ('Node', 'User', etc.) - generally the same string as
  105. * the destination class name without the MigrateDestination prefix.
  106. * @param $method
  107. * Method name such as 'prepare' (called at the beginning of an import operation)
  108. * or 'complete' (called at the end of an import operation).
  109. * @param ...
  110. * Parameters to be passed to the handler.
  111. */
  112. function migrate_handler_invoke_all($destination, $method) {
  113. $args = func_get_args();
  114. array_shift($args);
  115. array_shift($args);
  116. $return = array();
  117. $class_list = _migrate_class_list('MigrateDestinationHandler');
  118. $disabled = unserialize(variable_get('migrate_disabled_handlers', serialize(array())));
  119. foreach ($class_list as $class_name => $handler) {
  120. if (!in_array($class_name, $disabled) && $handler->handlesType($destination)
  121. && method_exists($handler, $method)) {
  122. migrate_instrument_start($class_name . '->' . $method);
  123. $result = call_user_func_array(array($handler, $method), $args);
  124. migrate_instrument_stop($class_name . '->' . $method);
  125. if (isset($result) && is_array($result)) {
  126. $return = array_merge($return, $result);
  127. }
  128. elseif (isset($result)) {
  129. $return[] = $result;
  130. }
  131. }
  132. }
  133. return $return;
  134. }
  135. /**
  136. * Invoke any available handlers attached to a given field type.
  137. *
  138. * @param $entity
  139. * The object we are building up before calling example_save().
  140. * @param $field_info
  141. * Array of info on the field, from field_info_field().
  142. * @param $instance
  143. * Array of info in the field instance, from field_info_instances().
  144. * @param $values
  145. * Array of incoming values, to be transformed into the appropriate structure
  146. * for the field type.
  147. * @param $method
  148. * Handler method to call (defaults to prepare()).
  149. */
  150. function migrate_field_handler_invoke_all($entity, array $field_info,
  151. array $instance, array $values, $method = 'prepare') {
  152. $return = array();
  153. $type = $field_info['type'];
  154. $class_list = _migrate_class_list('MigrateFieldHandler');
  155. $disabled = unserialize(variable_get('migrate_disabled_handlers',
  156. serialize(array())));
  157. $handler_called = FALSE;
  158. foreach ($class_list as $class_name => $handler) {
  159. if (!in_array($class_name, $disabled) && $handler->handlesType($type)
  160. && method_exists($handler, $method)) {
  161. migrate_instrument_start($class_name . '->' . $method);
  162. $result = call_user_func_array(array($handler, $method),
  163. array($entity, $field_info, $instance, $values));
  164. $handler_called = TRUE;
  165. migrate_instrument_stop($class_name . '->' . $method);
  166. if (isset($result) && is_array($result)) {
  167. $return = array_merge_recursive($return, $result);
  168. }
  169. elseif (isset($result)) {
  170. $return[] = $result;
  171. }
  172. }
  173. }
  174. if (!$handler_called && $method == 'prepare') {
  175. $handler = new MigrateDefaultFieldHandler();
  176. migrate_instrument_start('MigrateDefaultFieldHandler->prepare');
  177. $result = call_user_func_array(array($handler, 'prepare'),
  178. array($entity, $field_info, $instance, $values));
  179. migrate_instrument_stop('MigrateDefaultFieldHandler->prepare');
  180. if (isset($result) && is_array($result)) {
  181. $return = array_merge_recursive($return, $result);
  182. }
  183. elseif (isset($result)) {
  184. $return[] = $result;
  185. }
  186. }
  187. return $return;
  188. }
  189. /**
  190. * For a given parent class, identify and instantiate objects for any non-abstract
  191. * classes derived from the parent, returning an array of the objects indexed by
  192. * class name. The array will be ordered such that any classes with dependencies
  193. * are listed after the classes they are dependent on.
  194. *
  195. * @param $parent_class
  196. * Name of a class from which results will be derived.
  197. * @return
  198. * Array of objects, keyed by the class name.
  199. */
  200. function _migrate_class_list($parent_class) {
  201. // Get info on modules implementing Migrate API
  202. static $module_info;
  203. if (!isset($module_info)) {
  204. $module_info = migrate_get_module_apis();
  205. }
  206. static $class_lists = array();
  207. if (!isset($class_lists[$parent_class])) {
  208. $class_lists[$parent_class] = array();
  209. if ($parent_class == 'MigrateDestinationHandler') {
  210. $handler_key = 'destination handlers';
  211. }
  212. else {
  213. $handler_key = 'field handlers';
  214. }
  215. // Add explicitly-registered handler classes
  216. foreach ($module_info as $info) {
  217. if (isset($info[$handler_key]) && is_array($info[$handler_key])) {
  218. foreach ($info[$handler_key] as $handler_class) {
  219. $class_lists[$parent_class][$handler_class] = new $handler_class();
  220. }
  221. }
  222. }
  223. }
  224. return $class_lists[$parent_class];
  225. }
  226. /**
  227. * Implements hook_hook_info().
  228. */
  229. function migrate_hook_info() {
  230. // Look for hook_migrate_api() in example.migrate.inc.
  231. $hooks['migrate_api'] = array(
  232. 'group' => 'migrate',
  233. );
  234. $hooks['migrate_api_alter'] = array(
  235. 'group' => 'migrate',
  236. );
  237. return $hooks;
  238. }
  239. /**
  240. * Implementation of hook_permission().
  241. */
  242. function migrate_permission() {
  243. return array(
  244. MIGRATE_ACCESS_BASIC => array(
  245. 'title' => t('Access to basic migration information'),
  246. ),
  247. MIGRATE_ACCESS_ADVANCED => array(
  248. 'title' => t('Access to advanced migration information'),
  249. ),
  250. );
  251. }
  252. /**
  253. * Get a list of modules that support the current migrate API.
  254. */
  255. function migrate_get_module_apis($reset = FALSE) {
  256. static $cache = NULL;
  257. if ($reset) {
  258. $cache = NULL;
  259. }
  260. if (!isset($cache)) {
  261. $cache = array();
  262. foreach (module_implements('migrate_api') as $module) {
  263. $function = $module . '_migrate_api';
  264. $info = $function();
  265. if (isset($info['api']) && $info['api'] == MIGRATE_API_VERSION) {
  266. $cache[$module] = $info;
  267. }
  268. else {
  269. drupal_set_message(t('%function supports Migrate API version %modversion,
  270. Migrate module API version is %version - migration support not loaded.',
  271. array('%function' => $function, '%modversion' => $info['api'],
  272. '%version' => MIGRATE_API_VERSION)));
  273. }
  274. }
  275. // Allow modules to alter the migration information.
  276. drupal_alter('migrate_api', $cache);
  277. }
  278. return $cache;
  279. }
  280. /**
  281. * Register any migrations defined in hook_migrate_api().
  282. *
  283. * @param array $machine_names
  284. * If populated, only (re)register the specified migrations.
  285. */
  286. function migrate_static_registration($machine_names = array()) {
  287. $module_info = migrate_get_module_apis(TRUE);
  288. foreach ($module_info as $module => $info) {
  289. // Register any groups defined via the hook.
  290. if (isset($info['groups']) && is_array($info['groups'])) {
  291. foreach ($info['groups'] as $name => $arguments) {
  292. $title = $arguments['title'];
  293. unset($arguments['title']);
  294. MigrateGroup::register($name, $title, $arguments);
  295. }
  296. }
  297. // Register any migrations defined via the hook.
  298. if (isset($info['migrations']) && is_array($info['migrations'])) {
  299. foreach ($info['migrations'] as $machine_name => $arguments) {
  300. // If we have an explicit list to register, skip any not in the list.
  301. if (!empty($machine_names) && !in_array($machine_name, $machine_names)) {
  302. continue;
  303. }
  304. $class_name = $arguments['class_name'];
  305. unset($arguments['class_name']);
  306. // Call the right registerMigration implementation. Note that this means
  307. // that classes that override registerMigration() must always call it
  308. // directly, they cannot register those classes by defining them in
  309. // hook_migrate_api() and expect their extension to be called.
  310. if (is_subclass_of($class_name, 'Migration')) {
  311. Migration::registerMigration($class_name, $machine_name, $arguments);
  312. }
  313. else {
  314. MigrationBase::registerMigration($class_name, $machine_name, $arguments);
  315. }
  316. }
  317. }
  318. }
  319. }
  320. /**
  321. * Do a topological sort on our dependencies graph.
  322. */
  323. function migrate_order_dependencies($dependencies) {
  324. $visited = array();
  325. $list = array();
  326. foreach (array_keys($dependencies) as $name) {
  327. $visited[$name] = FALSE;
  328. }
  329. foreach (array_keys($dependencies) as $name) {
  330. migrate_visit_dependent($dependencies, $name, $list, $visited);
  331. }
  332. return $list;
  333. }
  334. /**
  335. * Depth-first search for independent migrations.
  336. */
  337. function migrate_visit_dependent($dependencies, $name, &$list, &$visited) {
  338. if ($visited[$name]) {
  339. if ($list[$name]) {
  340. return;
  341. }
  342. else {
  343. throw new MigrateException(t('Failure to sort migration list due to circular dependencies involving %name.', array('%name' => $name)));
  344. }
  345. }
  346. $visited[$name] = TRUE;
  347. if (isset($dependencies[$name])) {
  348. foreach ($dependencies[$name] as $dependent) {
  349. migrate_visit_dependent($dependencies, $dependent, $list, $visited);
  350. }
  351. }
  352. $list[$name] = $name;
  353. }
  354. /**
  355. * Implements hook_watchdog().
  356. * Find the migration that is currently running and notify it.
  357. *
  358. * @param array $log_entry
  359. */
  360. function migrate_watchdog($log_entry) {
  361. // Ensure that the Migration class exists, as different bootstrap phases may
  362. // not have included migration.inc yet.
  363. if (class_exists('Migration') && $migration = Migration::currentMigration()) {
  364. switch ($log_entry['severity']) {
  365. case WATCHDOG_EMERGENCY:
  366. case WATCHDOG_ALERT:
  367. case WATCHDOG_CRITICAL:
  368. case WATCHDOG_ERROR:
  369. $severity = MigrationBase::MESSAGE_ERROR;
  370. break;
  371. case WATCHDOG_WARNING:
  372. $severity = MigrationBase::MESSAGE_WARNING;
  373. break;
  374. case WATCHDOG_NOTICE:
  375. $severity = MigrationBase::MESSAGE_NOTICE;
  376. break;
  377. case WATCHDOG_DEBUG:
  378. case WATCHDOG_INFO:
  379. default:
  380. $severity = MigrationBase::MESSAGE_INFORMATIONAL;
  381. break;
  382. }
  383. $variables = is_array($log_entry['variables']) ? $log_entry['variables'] : array();
  384. $migration->saveMessage(t($log_entry['message'], $variables), $severity);
  385. }
  386. }
  387. /**
  388. * Resource functions modeled on Drupal's timer functions
  389. */
  390. /**
  391. * Save memory usage with the specified name. If you start and stop the same
  392. * memory name multiple times, the measured differences will be accumulated.
  393. *
  394. * @param name
  395. * The name of the memory measurement.
  396. */
  397. function migrate_memory_start($name) {
  398. global $_migrate_memory;
  399. $_migrate_memory[$name]['start'] = memory_get_usage();
  400. $_migrate_memory[$name]['count'] =
  401. isset($_migrate_memory[$name]['count']) ? ++$_migrate_memory[$name]['count'] : 1;
  402. }
  403. /**
  404. * Read the current memory value without recording the change.
  405. *
  406. * @param name
  407. * The name of the memory measurement.
  408. * @return
  409. * The change in bytes since the last start.
  410. */
  411. function migrate_memory_read($name) {
  412. global $_migrate_memory;
  413. if (isset($_migrate_memory[$name]['start'])) {
  414. $stop = memory_get_usage();
  415. $diff = $stop - $_migrate_memory[$name]['start'];
  416. if (isset($_migrate_memory[$name]['bytes'])) {
  417. $diff += $_migrate_memory[$name]['bytes'];
  418. }
  419. return $diff;
  420. }
  421. return $_migrate_memory[$name]['bytes'];
  422. }
  423. /**
  424. * Stop the memory counter with the specified name.
  425. *
  426. * @param name
  427. * The name of the memory measurement.
  428. * @return
  429. * A memory array. The array contains the number of times the memory has been
  430. * started and stopped (count) and the accumulated memory difference value in bytes.
  431. */
  432. function migrate_memory_stop($name) {
  433. global $_migrate_memory;
  434. if (isset($_migrate_memory[$name])) {
  435. if (isset($_migrate_memory[$name]['start'])) {
  436. $stop = memory_get_usage();
  437. $diff = $stop - $_migrate_memory[$name]['start'];
  438. if (isset($_migrate_memory[$name]['bytes'])) {
  439. $_migrate_memory[$name]['bytes'] += $diff;
  440. }
  441. else {
  442. $_migrate_memory[$name]['bytes'] = $diff;
  443. }
  444. unset($_migrate_memory[$name]['start']);
  445. }
  446. return $_migrate_memory[$name];
  447. }
  448. }
  449. /**
  450. * Start measuring time and (optionally) memory consumption over a section of code.
  451. * Note that the memory consumption measurement is generally not useful in
  452. * lower areas of the code, where data is being generated that will be freed
  453. * by the next call to the same area. For example, measuring the memory
  454. * consumption of db_query is not going to be helpful.
  455. *
  456. * @param $name
  457. * The name of the measurement.
  458. * @param $include_memory
  459. * Measure both memory and timers. Defaults to FALSE (timers only).
  460. */
  461. function migrate_instrument_start($name, $include_memory = FALSE) {
  462. global $_migrate_track_memory, $_migrate_track_timer;
  463. if ($_migrate_track_memory && $include_memory) {
  464. migrate_memory_start($name);
  465. }
  466. if ($_migrate_track_timer) {
  467. timer_start($name);
  468. }
  469. }
  470. /**
  471. * Stop measuring both memory and time consumption over a section of code.
  472. *
  473. * @param $name
  474. * The name of the measurement.
  475. */
  476. function migrate_instrument_stop($name) {
  477. global $_migrate_track_memory, $_migrate_track_timer;
  478. if ($_migrate_track_timer) {
  479. timer_stop($name);
  480. }
  481. if ($_migrate_track_memory) {
  482. migrate_memory_stop($name);
  483. }
  484. }
  485. /**
  486. * Call hook_migrate_overview for overall documentation on implemented migrations.
  487. */
  488. function migrate_overview() {
  489. $overview = '';
  490. $results = module_invoke_all('migrate_overview');
  491. foreach ($results as $result) {
  492. $overview .= $result . ' ';
  493. }
  494. return $overview;
  495. }
  496. /**
  497. * Implements hook_modules_enabled.
  498. */
  499. function migrate_modules_enabled($modules) {
  500. if (array_intersect($modules, module_implements('migrate_api'))) {
  501. migrate_static_registration();
  502. }
  503. }
  504. /**
  505. * Implements hook_module_implements_alter().
  506. */
  507. function migrate_module_implements_alter(&$implementation, $hook) {
  508. // Ensure that the Migration class exists, as different bootstrap phases may
  509. // not have included migration.inc yet.
  510. if (class_exists('Migration') && $migration = Migration::currentMigration()) {
  511. $disable_hooks = $migration->getDisableHooks();
  512. if (isset($disable_hooks[$hook])) {
  513. foreach ($disable_hooks[$hook] as $module) {
  514. unset($implementation[$module]);
  515. }
  516. }
  517. }
  518. }