migrate.install 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578
  1. <?php
  2. /**
  3. * @file
  4. * Migrate module installation
  5. */
  6. function migrate_schema() {
  7. $schema = array();
  8. $schema['migrate_status'] = migrate_schema_status();
  9. $schema['migrate_log'] = migrate_schema_log();
  10. $schema['migrate_group'] = migrate_schema_group();
  11. $schema['migrate_field_mapping'] = migrate_schema_field_mapping();
  12. return $schema;
  13. }
  14. function migrate_schema_status() {
  15. return array(
  16. 'description' => 'Status information for migrations',
  17. 'fields' => array(
  18. 'machine_name' => array(
  19. 'type' => 'varchar',
  20. 'length' => 255,
  21. 'not null' => TRUE,
  22. 'description' => 'Unique machine name for migration',
  23. ),
  24. 'class_name' => array(
  25. 'type' => 'varchar',
  26. 'length' => 255,
  27. 'not null' => TRUE,
  28. 'description' => 'Name of class to instantiate for this migration',
  29. ),
  30. 'group_name' => array(
  31. 'type' => 'varchar',
  32. 'length' => 255,
  33. 'not null' => TRUE,
  34. 'description' => 'Name of group containing migration',
  35. ),
  36. 'status' => array(
  37. 'type' => 'int',
  38. 'size' => 'tiny',
  39. 'unsigned' => TRUE,
  40. 'not null' => TRUE,
  41. 'default' => 0,
  42. 'description' => 'Current status of migration',
  43. ),
  44. 'highwater' => array(
  45. 'type' => 'varchar',
  46. 'length' => 255,
  47. 'not null' => TRUE,
  48. 'default' => '',
  49. 'description' => 'Highwater mark for detecting updated content',
  50. ),
  51. 'arguments' => array(
  52. 'type' => 'blob',
  53. 'not null' => FALSE,
  54. 'size' => 'big',
  55. 'serialize' => TRUE,
  56. 'description' => 'A serialized array of arguments to the migration constructor',
  57. ),
  58. ),
  59. 'primary key' => array('machine_name'),
  60. );
  61. }
  62. function migrate_schema_log() {
  63. return array(
  64. 'description' => 'History of migration processes',
  65. 'fields' => array(
  66. 'mlid' => array(
  67. 'type' => 'serial',
  68. 'unsigned' => TRUE,
  69. 'not null' => TRUE,
  70. 'description' => 'Primary key for migrate_log table',
  71. ),
  72. 'machine_name' => array(
  73. 'type' => 'varchar',
  74. 'length' => 255,
  75. 'not null' => TRUE,
  76. 'description' => 'Unique machine name for migration',
  77. ),
  78. 'process_type' => array(
  79. 'type' => 'int',
  80. 'size' => 'tiny',
  81. 'unsigned' => TRUE,
  82. 'not null' => TRUE,
  83. 'description' => 'Type of migration process - 1 for import, 2 for rollback',
  84. ),
  85. 'starttime' => array(
  86. 'type' => 'int',
  87. 'size' => 'big',
  88. 'unsigned' => TRUE,
  89. 'not null' => TRUE,
  90. 'description' => 'Begin time of a migration process, times 1000',
  91. ),
  92. 'endtime' => array(
  93. 'type' => 'int',
  94. 'size' => 'big',
  95. 'unsigned' => TRUE,
  96. 'not null' => FALSE,
  97. 'description' => 'End time of a migration process, times 1000',
  98. ),
  99. 'initialhighwater' => array(
  100. 'type' => 'varchar',
  101. 'length' => 255,
  102. 'not null' => TRUE,
  103. 'description' => 'Initial highwater mark',
  104. ),
  105. 'finalhighwater' => array(
  106. 'type' => 'varchar',
  107. 'length' => 255,
  108. 'not null' => FALSE,
  109. 'description' => 'Final highwater mark',
  110. ),
  111. 'numprocessed' => array(
  112. 'type' => 'int',
  113. 'unsigned' => TRUE,
  114. 'not null' => FALSE,
  115. 'description' => 'Number of items processed',
  116. ),
  117. ),
  118. 'primary key' => array('mlid'),
  119. );
  120. }
  121. function migrate_schema_group() {
  122. return array(
  123. 'description' => 'Information on migration groups',
  124. 'fields' => array(
  125. 'name' => array(
  126. 'type' => 'varchar',
  127. 'length' => 255,
  128. 'not null' => TRUE,
  129. 'description' => 'Unique machine name for a migration group',
  130. ),
  131. 'title' => array(
  132. 'type' => 'varchar',
  133. 'length' => 255,
  134. 'not null' => TRUE,
  135. 'description' => 'Display name for a migration group',
  136. ),
  137. 'arguments' => array(
  138. 'type' => 'blob',
  139. 'not null' => FALSE,
  140. 'size' => 'big',
  141. 'serialize' => TRUE,
  142. 'description' => 'A serialized array of arguments to the migration group',
  143. ),
  144. ),
  145. 'primary key' => array('name'),
  146. );
  147. }
  148. function migrate_schema_field_mapping() {
  149. return array(
  150. 'description' => 'History of migration processes',
  151. 'fields' => array(
  152. 'fmid' => array(
  153. 'type' => 'serial',
  154. 'unsigned' => TRUE,
  155. 'not null' => TRUE,
  156. 'description' => 'Unique ID for the field mapping row',
  157. ),
  158. 'machine_name' => array(
  159. 'type' => 'varchar',
  160. 'length' => 255,
  161. 'not null' => TRUE,
  162. 'description' => 'Parent migration for the field mapping',
  163. ),
  164. 'destination_field' => array(
  165. 'type' => 'varchar',
  166. 'length' => 255,
  167. 'not null' => TRUE,
  168. 'description' => 'Destination field for the field mapping',
  169. ),
  170. 'source_field' => array(
  171. 'type' => 'varchar',
  172. 'length' => 255,
  173. 'not null' => TRUE,
  174. 'description' => 'Source field for the field mapping',
  175. ),
  176. 'options' => array(
  177. 'type' => 'blob',
  178. 'not null' => FALSE,
  179. 'size' => 'big',
  180. 'serialize' => TRUE,
  181. 'description' => 'A serialized MigrateFieldMapping object holding all options',
  182. ),
  183. ),
  184. 'primary key' => array('fmid'),
  185. );
  186. }
  187. /**
  188. * Implements hook_uninstall().
  189. * Drop map/message tables, in case implementing classes did not.
  190. */
  191. function migrate_uninstall() {
  192. // Note: If a derived Migration class defined its own map or message
  193. // table name not fitting this pattern, that class is solely responsible for
  194. // cleaning up
  195. // TODO: Prefix table names (db_find_tables does not do it)
  196. foreach (db_find_tables('migrate_map_%') as $tablename) {
  197. db_drop_table($tablename);
  198. }
  199. foreach (db_find_tables('migrate_message_%') as $tablename) {
  200. db_drop_table($tablename);
  201. }
  202. // Remove any file_usage entries we've written
  203. if (db_table_exists('file_usage')) {
  204. db_delete('file_usage')
  205. ->condition('module', 'migrate')
  206. ->execute();
  207. }
  208. // Remove variables
  209. variable_del('migrate_disable_autoregistration');
  210. variable_del('migrate_disabled_handlers');
  211. variable_del('migrate_deprecation_warnings');
  212. }
  213. /**
  214. * Add highwater mark
  215. */
  216. function migrate_update_7001() {
  217. if (!db_field_exists('migrate_status', 'highwater')) {
  218. db_add_field('migrate_status', 'highwater', array(
  219. 'type' => 'varchar',
  220. 'length' => 255,
  221. 'not null' => TRUE,
  222. 'default' => '',
  223. 'description' => 'Highwater mark for detecting updated content',
  224. )
  225. );
  226. }
  227. $ret = t('Added highwater column to migrate_status table');
  228. return $ret;
  229. }
  230. /**
  231. * Add last_imported field to all map tables
  232. */
  233. function migrate_update_7002() {
  234. foreach (db_find_tables('migrate_map_%') as $tablename) {
  235. if (!db_field_exists($tablename, 'last_imported')) {
  236. db_add_field($tablename, 'last_imported', array(
  237. 'type' => 'int',
  238. 'unsigned' => TRUE,
  239. 'not null' => TRUE,
  240. 'default' => 0,
  241. 'description' => 'UNIX timestamp of the last time this row was imported',
  242. ));
  243. }
  244. }
  245. $ret = t('Added last_imported column to all map tables');
  246. return $ret;
  247. }
  248. /**
  249. * Add lastthroughput column to migrate_status
  250. */
  251. function migrate_update_7003() {
  252. $ret = '';
  253. if (!db_field_exists('migrate_status', 'lastthroughput')) {
  254. db_add_field('migrate_status', 'lastthroughput', array(
  255. 'type' => 'int',
  256. 'length' => 11,
  257. 'not null' => FALSE,
  258. 'description' => 'Rate of success during most recent completed import (# per minute)',
  259. )
  260. );
  261. }
  262. $ret = t('Added lastthroughput column to migrate_status table');
  263. return $ret;
  264. }
  265. /**
  266. * Convert lastimported datetime field to lastimportedtime int field.
  267. */
  268. function migrate_update_7004() {
  269. $ret = '';
  270. if (!db_field_exists('migrate_status', 'lastimportedtime')) {
  271. db_add_field('migrate_status', 'lastimportedtime', array(
  272. 'type' => 'int',
  273. 'unsigned' => TRUE,
  274. 'not null' => FALSE,
  275. 'description' => 'Date and time of last completed import',
  276. )
  277. );
  278. if (db_field_exists('migrate_status', 'lastimported')) {
  279. $result = db_select('migrate_status', 'ms')
  280. ->fields('ms', array('machine_name', 'lastimported'))
  281. ->execute();
  282. foreach ($result as $row) {
  283. $lastimportedtime = strtotime($row->lastimported);
  284. db_update('migrate_status')
  285. ->fields(array('lastimportedtime' => $lastimportedtime))
  286. ->condition('machine_name', $row->machine_name)
  287. ->execute();
  288. }
  289. db_drop_field('migrate_status', 'lastimported');
  290. $ret .= "\n" . t('Converted lastimported datetime field to lastimportedtime int field');
  291. }
  292. }
  293. return $ret;
  294. }
  295. /**
  296. * Add support for history logging
  297. */
  298. function migrate_update_7005() {
  299. $ret = '';
  300. if (!db_table_exists('migrate_log')) {
  301. $ret .= "\n" . t('Create migrate_log table');
  302. db_create_table('migrate_log', migrate_schema_log());
  303. $ret .= "\n" . t('Remove historic columns from migrate_status table');
  304. db_drop_field('migrate_status', 'lastthroughput');
  305. db_drop_field('migrate_status', 'lastimportedtime');
  306. }
  307. return $ret;
  308. }
  309. /**
  310. * Add and populate class_name field. Any existing migration code using
  311. * dependencies or sourceMigration() must be changed! See CHANGELOG.txt.
  312. */
  313. function migrate_update_7006() {
  314. $ret = '';
  315. if (!db_field_exists('migrate_status', 'class_name')) {
  316. db_add_field('migrate_status', 'class_name', array(
  317. 'type' => 'varchar',
  318. 'length' => 255,
  319. 'not null' => TRUE,
  320. 'default' => '',
  321. 'description' => 'Name of class to instantiate for this migration',
  322. )
  323. );
  324. db_query("UPDATE {migrate_status}
  325. SET class_name = CONCAT(machine_name, 'Migration')
  326. ");
  327. $ret = t('Added class_name column to migrate_status table');
  328. }
  329. return $ret;
  330. }
  331. /**
  332. * Add arguments field to migrate_status table.
  333. */
  334. function migrate_update_7007() {
  335. $ret = '';
  336. if (!db_field_exists('migrate_status', 'arguments')) {
  337. db_add_field('migrate_status', 'arguments', array(
  338. 'type' => 'blob',
  339. 'not null' => FALSE,
  340. 'size' => 'big',
  341. 'serialize' => TRUE,
  342. 'description' => 'A serialized array of arguments to the migration constructor',
  343. )
  344. );
  345. $ret = t('Added arguments column to migrate_status table');
  346. }
  347. return $ret;
  348. }
  349. /**
  350. * Update map tables to reflect change of needs_update to a status column.
  351. */
  352. function migrate_update_7008() {
  353. // Updates can be run when the module is disabled, which would mean the
  354. // call to migrate_migrations() will fail. Just bail in that case...
  355. if (!module_exists('migrate')) {
  356. throw new DrupalUpdateException(t('This update cannot be run while the Migrate module is disabled - you must enable Migrate to run this update.'));
  357. }
  358. $ret = '';
  359. foreach (migrate_migrations() as $migration) {
  360. if (is_a($migration, 'Migration')) {
  361. // Since we're now tracking failed/ignored rows in the map table,
  362. // destination keys need to be nullable
  363. $map = $migration->getMap();
  364. $map_connection = $map->getConnection();
  365. $map_table = $map->getMapTable();
  366. $destination = $migration->getDestination();
  367. $key_schema = $destination->getKeySchema();
  368. $index = 1;
  369. foreach ($key_schema as $field_schema) {
  370. $field = 'destid' . $index++;
  371. $field_schema['not null'] = FALSE;
  372. $map_connection->schema()->changeField($map_table, $field, $field,
  373. $field_schema);
  374. $ret .= "\n" . t('Changed !table.!field to be non-null',
  375. array('!table' => $map_table, '!field' => $field));
  376. }
  377. // Add any existing failures to the map table
  378. $msg_table = $map->getMessageTable();
  379. $msg_marked = FALSE;
  380. $result = $map_connection->select($msg_table, 'msg')
  381. ->fields('msg')
  382. ->condition('level', Migration::MESSAGE_INFORMATIONAL, '<>')
  383. ->execute();
  384. foreach ($result as $row) {
  385. $keys = array();
  386. $index = 1;
  387. foreach ($row as $field => $value) {
  388. if (drupal_substr($field, 0, 8) == 'sourceid') {
  389. $keys['sourceid' . $index++] = $value;
  390. }
  391. }
  392. $map_connection->merge($map_table)
  393. ->key($keys)
  394. ->fields(array('needs_update' => MigrateMap::STATUS_FAILED))
  395. ->execute();
  396. $msg_marked = TRUE;
  397. }
  398. if ($msg_marked) {
  399. $ret .= "\n" . t('Marked failures in !table', array('!table' => $map_table));
  400. }
  401. }
  402. }
  403. return $ret;
  404. }
  405. /**
  406. * Warn that there have been incompatible changes to file handling.
  407. */
  408. function migrate_update_7201() {
  409. return t('File field and destination handling has been completely refactored
  410. - if you are migrating files, you will need to change your migration
  411. implementation to reflect these changes. Please see
  412. <a href="@doc">Handling files in Drupal 7</a> for more information',
  413. array('@doc' => 'http://drupal.org/node/1540106'));
  414. }
  415. /**
  416. * Add rollback_action field to all map tables in the Drupal database.
  417. */
  418. function migrate_update_7202() {
  419. // Note this won't catch any prefixed tables, or any stored in the source
  420. // database - ensureTables() will take care of those.
  421. foreach (db_find_tables('migrate_map_%') as $tablename) {
  422. if (!db_field_exists($tablename, 'rollback_action')) {
  423. db_add_field($tablename, 'rollback_action', array(
  424. 'type' => 'int',
  425. 'size' => 'tiny',
  426. 'unsigned' => TRUE,
  427. 'not null' => TRUE,
  428. 'default' => 0,
  429. 'description' => 'Flag indicating what to do for this item on rollback',
  430. ));
  431. }
  432. }
  433. $ret = t('Added rollback_action column to all map tables');
  434. return $ret;
  435. }
  436. /**
  437. * Add database tracking of per-group info.
  438. */
  439. function migrate_update_7203() {
  440. $ret = '';
  441. if (!db_table_exists('migrate_group')) {
  442. $ret .= t('Create migrate_group table') . "\n";
  443. db_create_table('migrate_group', migrate_schema_group());
  444. }
  445. if (!db_field_exists('migrate_status', 'group_name')) {
  446. $ret .= t('Add group relationship to migrate_status table') . "\n";
  447. db_add_field('migrate_status', 'group_name', array(
  448. 'type' => 'varchar',
  449. 'length' => 255,
  450. 'not null' => TRUE,
  451. 'default' => 'default',
  452. 'description' => 'Name of group containing migration',
  453. )
  454. );
  455. // Populate each migration's group_name field
  456. $groups = array();
  457. foreach (migrate_migrations() as $machine_name => $migration) {
  458. $group_name = $migration->getGroup()->getName();
  459. if (empty($group_name)) {
  460. $group_name = 'default';
  461. }
  462. $groups[$group_name] = $group_name;
  463. db_update('migrate_status')
  464. ->fields(array('group_name' => $group_name))
  465. ->condition('machine_name', $machine_name)
  466. ->execute();
  467. }
  468. // Populate the migrate_group table
  469. foreach ($groups as $group_name) {
  470. $title = db_select('migrate_group', 'mg')
  471. ->fields('mg', array('title'))
  472. ->condition('name', $group_name)
  473. ->execute()
  474. ->fetchField();
  475. if (!$title) {
  476. db_insert('migrate_group')
  477. ->fields(array(
  478. 'name' => $group_name,
  479. 'title' => $group_name,
  480. 'arguments' => serialize(array()),
  481. ))
  482. ->execute();
  483. }
  484. }
  485. }
  486. return $ret;
  487. }
  488. /**
  489. * Add database tracking of field mappings.
  490. */
  491. function migrate_update_7204() {
  492. $ret = '';
  493. if (!db_table_exists('migrate_field_mapping')) {
  494. $ret = t('Create migrate_field_mapping table');
  495. db_create_table('migrate_field_mapping', migrate_schema_field_mapping());
  496. }
  497. return $ret;
  498. }
  499. /**
  500. * Remove obsolete autoregistration disablement.
  501. */
  502. function migrate_update_7205() {
  503. variable_del('migrate_disable_autoregistration');
  504. }
  505. /**
  506. * Replace three-column PK with a simple serial.
  507. */
  508. function migrate_update_7206() {
  509. if (!db_field_exists('migrate_field_mapping', 'fmid')) {
  510. db_drop_primary_key('migrate_field_mapping');
  511. db_add_field('migrate_field_mapping', 'fmid',
  512. array(
  513. 'type' => 'serial',
  514. 'unsigned' => TRUE,
  515. 'not null' => TRUE,
  516. 'description' => 'Unique ID for the field mapping row',
  517. ),
  518. array(
  519. 'primary key' => array('fmid'),
  520. )
  521. );
  522. }
  523. }
  524. /**
  525. * Make sure we remove an empty 'default' group created by the previous updates.
  526. */
  527. function migrate_update_7207() {
  528. $rows = db_select('migrate_group', 'mg')
  529. ->fields('mg', array('name'))
  530. ->condition('name', 'default')
  531. ->execute()
  532. ->rowCount();
  533. if ($rows > 0) {
  534. $rows = db_select('migrate_status', 'ms')
  535. ->fields('ms', array('machine_name'))
  536. ->condition('group_name', 'default')
  537. ->execute()
  538. ->rowCount();
  539. if ($rows == 0) {
  540. db_delete('migrate_group')
  541. ->condition('name', 'default')
  542. ->execute();
  543. }
  544. }
  545. }