migrate.module 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. <?php
  2. /**
  3. * @file
  4. * API and drush commands to support migration of data from external sources
  5. * into a Drupal installation.
  6. */
  7. // TODO:
  8. // Continue hook_schema_alter() for map & message tables?
  9. // Views hooks for map/message tables
  10. // xlat support?
  11. // Documentation
  12. // Tests
  13. define('MIGRATE_API_VERSION', 2);
  14. define('MIGRATE_ACCESS_BASIC', 'migration information');
  15. define('MIGRATE_ACCESS_ADVANCED', 'advanced migration information');
  16. /**
  17. * Retrieve a list of all active migrations, ordered by dependencies. To be
  18. * recognized, a class must be non-abstract, and derived from MigrationBase.
  19. *
  20. * @param $reset
  21. * If TRUE, the static cache of migrations will be flushed before attempting to
  22. * reinstantiate all active migrations. This can be important for script runs
  23. * where migration classes may be dynamically registered.
  24. *
  25. * @return
  26. * Array of migration objects, keyed by the machine name.
  27. */
  28. function migrate_migrations($reset = NULL) {
  29. static $migrations = array();
  30. if (!empty($migrations) && empty($reset)) {
  31. return $migrations;
  32. }
  33. // Get list of modules implementing Migrate API - mainly, we're looking to
  34. // make sure any dynamic migrations defined in hook_migrate_api() get registered.
  35. migrate_get_module_apis(TRUE);
  36. $dependencies_list = array();
  37. $dependent_migrations = array();
  38. $required_migrations = array();
  39. $result = db_select('migrate_status', 'ms')
  40. ->fields('ms', array('machine_name', 'class_name'))
  41. ->execute();
  42. foreach ($result as $row) {
  43. if (class_exists($row->class_name)) {
  44. $reflect = new ReflectionClass($row->class_name);
  45. if (!$reflect->isAbstract() && $reflect->isSubclassOf('MigrationBase')) {
  46. $migration = MigrationBase::getInstance($row->machine_name);
  47. if ($migration) {
  48. $dependencies = $migration->getDependencies();
  49. $dependencies_list[$row->machine_name] = $dependencies;
  50. if (count($dependencies) > 0) {
  51. // Set classes with dependencies aside for reordering
  52. $dependent_migrations[$row->machine_name] = $migration;
  53. $required_migrations += $dependencies;
  54. }
  55. else {
  56. // No dependencies, just add
  57. $migrations[$row->machine_name] = $migration;
  58. }
  59. }
  60. }
  61. else {
  62. MigrationBase::displayMessage(t('Class !class is no longer a valid concrete migration class',
  63. array('!class' => $row->class_name)));
  64. }
  65. }
  66. else {
  67. MigrationBase::displayMessage(t('Class !class no longer exists',
  68. array('!class' => $row->class_name)));
  69. }
  70. }
  71. $ordered_migrations = migrate_order_dependencies($dependencies_list);
  72. foreach ($ordered_migrations as $name) {
  73. if (!isset($migrations[$name])) {
  74. $migrations[$name] = $dependent_migrations[$name];
  75. }
  76. }
  77. // The migrations are now ordered according to their own dependencies - now order
  78. // them by group
  79. $groups = MigrateGroup::groups();
  80. // Seed the final list by properly-ordered groups.
  81. $final_migrations = array();
  82. foreach ($groups as $name => $group) {
  83. $final_migrations[$name] = array();
  84. }
  85. // Fill in the grouped list.
  86. foreach ($migrations as $machine_name => $migration) {
  87. if (!method_exists($migration, 'getGroup')) {
  88. MigrationBase::displayMessage(t('Migration !machine_name is not a valid Migration dependency.', array(
  89. '!machine_name' => $machine_name,
  90. )));
  91. }
  92. else {
  93. $final_migrations[$migration->getGroup()->getName()][$machine_name] = $migration;
  94. }
  95. }
  96. // Flatten the grouped list.
  97. $migrations = array();
  98. foreach ($final_migrations as $group_name => $group_migrations) {
  99. foreach ($group_migrations as $machine_name => $migration) {
  100. $migrations[$machine_name] = $migration;
  101. }
  102. }
  103. return $migrations;
  104. }
  105. /**
  106. * Invoke any available handlers attached to a given destination type.
  107. * If any handlers have dependencies defined, they will be invoked after
  108. * the specified handlers.
  109. *
  110. * @param $destination
  111. * Destination type ('Node', 'User', etc.) - generally the same string as
  112. * the destination class name without the MigrateDestination prefix.
  113. * @param $method
  114. * Method name such as 'prepare' (called at the beginning of an import operation)
  115. * or 'complete' (called at the end of an import operation).
  116. * @param ...
  117. * Parameters to be passed to the handler.
  118. */
  119. function migrate_handler_invoke_all($destination, $method) {
  120. $args = func_get_args();
  121. array_shift($args);
  122. array_shift($args);
  123. $return = array();
  124. $class_list = _migrate_class_list('MigrateDestinationHandler');
  125. $disabled = unserialize(variable_get('migrate_disabled_handlers', serialize(array())));
  126. foreach ($class_list as $class_name => $handler) {
  127. if (!in_array($class_name, $disabled) && $handler->handlesType($destination)
  128. && method_exists($handler, $method)) {
  129. migrate_instrument_start($class_name . '->' . $method);
  130. $result = call_user_func_array(array($handler, $method), $args);
  131. migrate_instrument_stop($class_name . '->' . $method);
  132. if (isset($result) && is_array($result)) {
  133. $return = array_merge($return, $result);
  134. }
  135. elseif (isset($result)) {
  136. $return[] = $result;
  137. }
  138. }
  139. }
  140. return $return;
  141. }
  142. /**
  143. * Invoke any available handlers attached to a given field type.
  144. *
  145. * @param $entity
  146. * The object we are building up before calling example_save().
  147. * @param $field_info
  148. * Array of info on the field, from field_info_field().
  149. * @param $instance
  150. * Array of info in the field instance, from field_info_instances().
  151. * @param $values
  152. * Array of incoming values, to be transformed into the appropriate structure
  153. * for the field type.
  154. * @param $method
  155. * Handler method to call (defaults to prepare()).
  156. */
  157. function migrate_field_handler_invoke_all($entity, array $field_info,
  158. array $instance, array $values, $method = 'prepare') {
  159. $return = array();
  160. $type = $field_info['type'];
  161. $class_list = _migrate_class_list('MigrateFieldHandler');
  162. $disabled = unserialize(variable_get('migrate_disabled_handlers',
  163. serialize(array())));
  164. $handler_called = FALSE;
  165. foreach ($class_list as $class_name => $handler) {
  166. if (!in_array($class_name, $disabled) && $handler->handlesType($type)
  167. && method_exists($handler, $method)) {
  168. migrate_instrument_start($class_name . '->' . $method);
  169. $result = call_user_func_array(array($handler, $method),
  170. array($entity, $field_info, $instance, $values));
  171. $handler_called = TRUE;
  172. migrate_instrument_stop($class_name . '->' . $method);
  173. if (isset($result) && is_array($result)) {
  174. $return = array_merge_recursive($return, $result);
  175. }
  176. elseif (isset($result)) {
  177. $return[] = $result;
  178. }
  179. }
  180. }
  181. if (!$handler_called && $method == 'prepare') {
  182. $handler = new MigrateDefaultFieldHandler();
  183. migrate_instrument_start('MigrateDefaultFieldHandler->prepare');
  184. $result = call_user_func_array(array($handler, 'prepare'),
  185. array($entity, $field_info, $instance, $values));
  186. migrate_instrument_stop('MigrateDefaultFieldHandler->prepare');
  187. if (isset($result) && is_array($result)) {
  188. $return = array_merge_recursive($return, $result);
  189. }
  190. elseif (isset($result)) {
  191. $return[] = $result;
  192. }
  193. }
  194. return $return;
  195. }
  196. /**
  197. * For a given parent class, identify and instantiate objects for any non-abstract
  198. * classes derived from the parent, returning an array of the objects indexed by
  199. * class name. The array will be ordered such that any classes with dependencies
  200. * are listed after the classes they are dependent on.
  201. *
  202. * @param $parent_class
  203. * Name of a class from which results will be derived.
  204. * @return
  205. * Array of objects, keyed by the class name.
  206. */
  207. function _migrate_class_list($parent_class) {
  208. // Get info on modules implementing Migrate API
  209. static $module_info;
  210. if (!isset($module_info)) {
  211. $module_info = migrate_get_module_apis();
  212. }
  213. static $class_lists = array();
  214. if (!isset($class_lists[$parent_class])) {
  215. $class_lists[$parent_class] = array();
  216. if ($parent_class == 'MigrateDestinationHandler') {
  217. $handler_key = 'destination handlers';
  218. }
  219. else {
  220. $handler_key = 'field handlers';
  221. }
  222. // Add explicitly-registered handler classes
  223. foreach ($module_info as $info) {
  224. if (isset($info[$handler_key]) && is_array($info[$handler_key])) {
  225. foreach ($info[$handler_key] as $handler_class) {
  226. $class_lists[$parent_class][$handler_class] = new $handler_class();
  227. }
  228. }
  229. }
  230. }
  231. return $class_lists[$parent_class];
  232. }
  233. /**
  234. * Implements hook_hook_info().
  235. */
  236. function migrate_hook_info() {
  237. // Look for hook_migrate_api() in example.migrate.inc.
  238. $hooks['migrate_api'] = array(
  239. 'group' => 'migrate',
  240. );
  241. $hooks['migrate_api_alter'] = array(
  242. 'group' => 'migrate',
  243. );
  244. return $hooks;
  245. }
  246. /**
  247. * Implementation of hook_permission().
  248. */
  249. function migrate_permission() {
  250. return array(
  251. MIGRATE_ACCESS_BASIC => array(
  252. 'title' => t('Access to basic migration information'),
  253. ),
  254. MIGRATE_ACCESS_ADVANCED => array(
  255. 'title' => t('Access to advanced migration information'),
  256. ),
  257. );
  258. }
  259. /**
  260. * Get a list of modules that support the current migrate API.
  261. */
  262. function migrate_get_module_apis($reset = FALSE) {
  263. static $cache = NULL;
  264. if ($reset) {
  265. $cache = NULL;
  266. }
  267. if (!isset($cache)) {
  268. $cache = array();
  269. foreach (module_implements('migrate_api') as $module) {
  270. $function = $module . '_migrate_api';
  271. $info = $function();
  272. if (isset($info['api']) && $info['api'] == MIGRATE_API_VERSION) {
  273. $cache[$module] = $info;
  274. }
  275. else {
  276. drupal_set_message(t('%function supports Migrate API version %modversion,
  277. Migrate module API version is %version - migration support not loaded.',
  278. array('%function' => $function, '%modversion' => $info['api'],
  279. '%version' => MIGRATE_API_VERSION)));
  280. }
  281. }
  282. // Allow modules to alter the migration information.
  283. drupal_alter('migrate_api', $cache);
  284. }
  285. return $cache;
  286. }
  287. /**
  288. * Register any migrations defined in hook_migrate_api().
  289. *
  290. * @param array $machine_names
  291. * If populated, only (re)register the specified migrations.
  292. */
  293. function migrate_static_registration($machine_names = array()) {
  294. $module_info = migrate_get_module_apis(TRUE);
  295. foreach ($module_info as $module => $info) {
  296. // Register any groups defined via the hook.
  297. if (isset($info['groups']) && is_array($info['groups'])) {
  298. foreach ($info['groups'] as $name => $arguments) {
  299. $title = $arguments['title'];
  300. unset($arguments['title']);
  301. MigrateGroup::register($name, $title, $arguments);
  302. }
  303. }
  304. // Register any migrations defined via the hook.
  305. if (isset($info['migrations']) && is_array($info['migrations'])) {
  306. foreach ($info['migrations'] as $machine_name => $arguments) {
  307. // If we have an explicit list to register, skip any not in the list.
  308. if (!empty($machine_names) && !in_array($machine_name, $machine_names)) {
  309. continue;
  310. }
  311. $class_name = $arguments['class_name'];
  312. unset($arguments['class_name']);
  313. // Call the right registerMigration implementation. Note that this means
  314. // that classes that override registerMigration() must always call it
  315. // directly, they cannot register those classes by defining them in
  316. // hook_migrate_api() and expect their extension to be called.
  317. if (is_subclass_of($class_name, 'Migration')) {
  318. Migration::registerMigration($class_name, $machine_name, $arguments);
  319. }
  320. else {
  321. MigrationBase::registerMigration($class_name, $machine_name, $arguments);
  322. }
  323. }
  324. }
  325. }
  326. }
  327. /**
  328. * Do a topological sort on our dependencies graph.
  329. */
  330. function migrate_order_dependencies($dependencies) {
  331. $visited = array();
  332. $list = array();
  333. foreach (array_keys($dependencies) as $name) {
  334. $visited[$name] = FALSE;
  335. }
  336. foreach (array_keys($dependencies) as $name) {
  337. migrate_visit_dependent($dependencies, $name, $list, $visited);
  338. }
  339. return $list;
  340. }
  341. /**
  342. * Depth-first search for independent migrations.
  343. */
  344. function migrate_visit_dependent($dependencies, $name, &$list, &$visited) {
  345. if ($visited[$name]) {
  346. if ($list[$name]) {
  347. return;
  348. }
  349. else {
  350. throw new MigrateException(t('Failure to sort migration list due to circular dependencies involving %name.', array('%name' => $name)));
  351. }
  352. }
  353. $visited[$name] = TRUE;
  354. if (isset($dependencies[$name])) {
  355. foreach ($dependencies[$name] as $dependent) {
  356. migrate_visit_dependent($dependencies, $dependent, $list, $visited);
  357. }
  358. }
  359. $list[$name] = $name;
  360. }
  361. /**
  362. * Implements hook_watchdog().
  363. * Find the migration that is currently running and notify it.
  364. *
  365. * @param array $log_entry
  366. */
  367. function migrate_watchdog($log_entry) {
  368. // Ensure that the Migration class exists, as different bootstrap phases may
  369. // not have included migration.inc yet.
  370. if (class_exists('Migration') && $migration = Migration::currentMigration()) {
  371. switch ($log_entry['severity']) {
  372. case WATCHDOG_EMERGENCY:
  373. case WATCHDOG_ALERT:
  374. case WATCHDOG_CRITICAL:
  375. case WATCHDOG_ERROR:
  376. $severity = MigrationBase::MESSAGE_ERROR;
  377. break;
  378. case WATCHDOG_WARNING:
  379. $severity = MigrationBase::MESSAGE_WARNING;
  380. break;
  381. case WATCHDOG_NOTICE:
  382. $severity = MigrationBase::MESSAGE_NOTICE;
  383. break;
  384. case WATCHDOG_DEBUG:
  385. case WATCHDOG_INFO:
  386. default:
  387. $severity = MigrationBase::MESSAGE_INFORMATIONAL;
  388. break;
  389. }
  390. $variables = is_array($log_entry['variables']) ? $log_entry['variables'] : array();
  391. $migration->saveMessage(t($log_entry['message'], $variables), $severity);
  392. }
  393. }
  394. /**
  395. * Resource functions modeled on Drupal's timer functions
  396. */
  397. /**
  398. * Save memory usage with the specified name. If you start and stop the same
  399. * memory name multiple times, the measured differences will be accumulated.
  400. *
  401. * @param name
  402. * The name of the memory measurement.
  403. */
  404. function migrate_memory_start($name) {
  405. global $_migrate_memory;
  406. $_migrate_memory[$name]['start'] = memory_get_usage();
  407. $_migrate_memory[$name]['count'] =
  408. isset($_migrate_memory[$name]['count']) ? ++$_migrate_memory[$name]['count'] : 1;
  409. }
  410. /**
  411. * Read the current memory value without recording the change.
  412. *
  413. * @param name
  414. * The name of the memory measurement.
  415. * @return
  416. * The change in bytes since the last start.
  417. */
  418. function migrate_memory_read($name) {
  419. global $_migrate_memory;
  420. if (isset($_migrate_memory[$name]['start'])) {
  421. $stop = memory_get_usage();
  422. $diff = $stop - $_migrate_memory[$name]['start'];
  423. if (isset($_migrate_memory[$name]['bytes'])) {
  424. $diff += $_migrate_memory[$name]['bytes'];
  425. }
  426. return $diff;
  427. }
  428. return $_migrate_memory[$name]['bytes'];
  429. }
  430. /**
  431. * Stop the memory counter with the specified name.
  432. *
  433. * @param name
  434. * The name of the memory measurement.
  435. * @return
  436. * A memory array. The array contains the number of times the memory has been
  437. * started and stopped (count) and the accumulated memory difference value in bytes.
  438. */
  439. function migrate_memory_stop($name) {
  440. global $_migrate_memory;
  441. if (isset($_migrate_memory[$name])) {
  442. if (isset($_migrate_memory[$name]['start'])) {
  443. $stop = memory_get_usage();
  444. $diff = $stop - $_migrate_memory[$name]['start'];
  445. if (isset($_migrate_memory[$name]['bytes'])) {
  446. $_migrate_memory[$name]['bytes'] += $diff;
  447. }
  448. else {
  449. $_migrate_memory[$name]['bytes'] = $diff;
  450. }
  451. unset($_migrate_memory[$name]['start']);
  452. }
  453. return $_migrate_memory[$name];
  454. }
  455. }
  456. /**
  457. * Start measuring time and (optionally) memory consumption over a section of code.
  458. * Note that the memory consumption measurement is generally not useful in
  459. * lower areas of the code, where data is being generated that will be freed
  460. * by the next call to the same area. For example, measuring the memory
  461. * consumption of db_query is not going to be helpful.
  462. *
  463. * @param $name
  464. * The name of the measurement.
  465. * @param $include_memory
  466. * Measure both memory and timers. Defaults to FALSE (timers only).
  467. */
  468. function migrate_instrument_start($name, $include_memory = FALSE) {
  469. global $_migrate_track_memory, $_migrate_track_timer;
  470. if ($_migrate_track_memory && $include_memory) {
  471. migrate_memory_start($name);
  472. }
  473. if ($_migrate_track_timer) {
  474. timer_start($name);
  475. }
  476. }
  477. /**
  478. * Stop measuring both memory and time consumption over a section of code.
  479. *
  480. * @param $name
  481. * The name of the measurement.
  482. */
  483. function migrate_instrument_stop($name) {
  484. global $_migrate_track_memory, $_migrate_track_timer;
  485. if ($_migrate_track_timer) {
  486. timer_stop($name);
  487. }
  488. if ($_migrate_track_memory) {
  489. migrate_memory_stop($name);
  490. }
  491. }
  492. /**
  493. * Call hook_migrate_overview for overall documentation on implemented migrations.
  494. */
  495. function migrate_overview() {
  496. $overview = '';
  497. $results = module_invoke_all('migrate_overview');
  498. foreach ($results as $result) {
  499. $overview .= $result . ' ';
  500. }
  501. return $overview;
  502. }
  503. /**
  504. * Implements hook_modules_enabled.
  505. */
  506. function migrate_modules_enabled($modules) {
  507. if (array_intersect($modules, module_implements('migrate_api'))) {
  508. migrate_static_registration();
  509. }
  510. }
  511. /**
  512. * Implements hook_module_implements_alter().
  513. */
  514. function migrate_module_implements_alter(&$implementation, $hook) {
  515. // Ensure that the Migration class exists, as different bootstrap phases may
  516. // not have included migration.inc yet.
  517. if (class_exists('Migration') && $migration = Migration::currentMigration()) {
  518. $disable_hooks = $migration->getDisableHooks();
  519. if (isset($disable_hooks[$hook])) {
  520. foreach ($disable_hooks[$hook] as $module) {
  521. unset($implementation[$module]);
  522. }
  523. }
  524. }
  525. }