upgrades core to 8.4.2

core/modules/migrate/src/Annotation/MigrateDestination.php

@@ -43,4 +43,15 @@ class MigrateDestination extends Plugin {
    */
   public $requirements_met = TRUE;
 
+  /**
+   * Identifies the system handling the data the destination plugin will write.
+   *
+   * The destination plugin itself determines how the value is used. For
+   * example, Migrate Drupal's destination plugins expect destination_module to
+   * be the name of a module that must be installed on the destination.
+   *
+   * @var string
+   */
+  public $destination_module;
+
 }

core/modules/migrate/src/Annotation/MigrateSource.php

@@ -43,16 +43,15 @@ class MigrateSource extends Plugin implements MultipleProviderAnnotationInterfac
   /**
    * Identifies the system providing the data the source plugin will read.
    *
-   * This can be any type, and the source plugin itself determines how the value
-   * is used. For example, Migrate Drupal's source plugins expect
-   * source_provider to be the name of a module that must be installed and
-   * enabled in the source database.
+   * The source plugin itself determines how the value is used. For example,
+   * Migrate Drupal's source plugins expect source_module to be the name of a
+   * module that must be installed and enabled in the source database.
    *
    * @see \Drupal\migrate_drupal\Plugin\migrate\source\DrupalSqlBase::checkRequirements
    *
-   * @var mixed
+   * @var string
    */
-  public $source_provider;
+  public $source_module;
 
   /**
    * Specifies the minimum version of the source provider.
@@ -60,7 +59,7 @@ class MigrateSource extends Plugin implements MultipleProviderAnnotationInterfac
    * This can be any type, and the source plugin itself determines how it is
    * used. For example, Migrate Drupal's source plugins expect this to be an
    * integer representing the minimum installed database schema version of the
-   * module specified by source_provider.
+   * module specified by source_module.
    *
    * @var mixed
    */

core/modules/migrate/src/MigrateBuildDependencyInterface.php

@@ -2,7 +2,6 @@
 
 namespace Drupal\migrate;
 
-
 interface MigrateBuildDependencyInterface {
 
   /**

core/modules/migrate/src/MigrateExecutable.php

@@ -96,7 +96,7 @@ class MigrateExecutable implements MigrateExecutableInterface {
    * @param \Drupal\migrate\Plugin\MigrationInterface $migration
    *   The migration to run.
    * @param \Drupal\migrate\MigrateMessageInterface $message
-   *   The message to record.
+   *   The migrate message service.
    * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $event_dispatcher
    *   The event dispatcher.
    *
@@ -386,9 +386,14 @@ class MigrateExecutable implements MigrateExecutableInterface {
           $multiple = $plugin->multiple();
         }
       }
-      // No plugins or no value means do not set.
-      if ($plugins && !is_null($value)) {
-        $row->setDestinationProperty($destination, $value);
+      // Ensure all values, including nulls, are migrated.
+      if ($plugins) {
+        if (isset($value)) {
+          $row->setDestinationProperty($destination, $value);
+        }
+        else {
+          $row->setEmptyDestinationProperty($destination);
+        }
       }
       // Reset the value.
       $value = NULL;
@@ -534,6 +539,9 @@ class MigrateExecutable implements MigrateExecutableInterface {
 
     // @TODO: explore resetting the container.
 
+    // Run garbage collector to further reduce memory.
+    gc_collect_cycles();
+
     return memory_get_usage();
   }
 

core/modules/migrate/src/MigrateMessageInterface.php

@@ -2,7 +2,6 @@
 
 namespace Drupal\migrate;
 
-
 interface MigrateMessageInterface {
 
   /**

core/modules/migrate/src/Plugin/MigrateDestinationInterface.php

@@ -133,4 +133,12 @@ interface MigrateDestinationInterface extends PluginInspectionInterface {
    */
   public function rollbackAction();
 
+  /**
+   * Gets the destination module handling the destination data.
+   *
+   * @return string|null
+   *   The destination module or NULL if not found.
+   */
+  public function getDestinationModule();
+
 }

core/modules/migrate/src/Plugin/MigrateIdMapInterface.php

@@ -213,6 +213,8 @@ interface MigrateIdMapInterface extends \Iterator, PluginInspectionInterface {
    *
    * @deprecated in Drupal 8.1.x, will be removed before Drupal 9.0.x. Use
    *   lookupDestinationIds() instead.
+   *
+   * @see https://www.drupal.org/node/2725809
    */
   public function lookupDestinationId(array $source_id_values);
 
@@ -227,7 +229,7 @@ interface MigrateIdMapInterface extends \Iterator, PluginInspectionInterface {
    *   If unkeyed, the first count($source_id_values) keys will be assumed.
    *
    * @return array
-   *    An array of arrays of destination identifier values.
+   *   An array of arrays of destination identifier values.
    *
    * @throws \Drupal\migrate\MigrateException
    *   Thrown when $source_id_values contains unknown keys, or is the wrong
@@ -266,10 +268,10 @@ interface MigrateIdMapInterface extends \Iterator, PluginInspectionInterface {
   public function getQualifiedMapTableName();
 
   /**
-   * Sets the migrate message.
+   * Sets the migrate message service.
    *
    * @param \Drupal\migrate\MigrateMessageInterface $message
-   *   The message to display.
+   *   The migrate message service.
    */
   public function setMessage(MigrateMessageInterface $message);
 

core/modules/migrate/src/Plugin/MigrateSourceInterface.php

@@ -100,4 +100,12 @@ interface MigrateSourceInterface extends \Countable, \Iterator, PluginInspection
    */
   public function getIds();
 
+  /**
+   * Gets the source module providing the source data.
+   *
+   * @return string|null
+   *   The source module or NULL if not found.
+   */
+  public function getSourceModule();
+
 }

core/modules/migrate/src/Plugin/MigrateSourcePluginManager.php

@@ -12,7 +12,7 @@ use Drupal\migrate\Plugin\Discovery\ProviderFilterDecorator;
  * Plugin manager for migrate source plugins.
  *
  * @see \Drupal\migrate\Plugin\MigrateSourceInterface
- * @see \Drupal\migrate\Plugin\source\SourcePluginBase
+ * @see \Drupal\migrate\Plugin\migrate\source\SourcePluginBase
  * @see \Drupal\migrate\Annotation\MigrateSource
  * @see plugin_api
  *

core/modules/migrate/src/Plugin/Migration.php

@@ -312,6 +312,8 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
    *
    * @deprecated in Drupal 8.1.x, will be removed before Drupal 9.0.x. Use
    *   more specific getters instead.
+   *
+   * @see https://www.drupal.org/node/2873795
    */
   public function get($property) {
     return isset($this->$property) ? $this->$property : NULL;
@@ -570,7 +572,7 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
    */
   public function mergeProcessOfProperty($property, array $process_of_property) {
     // If we already have a process value then merge the incoming process array
-    //otherwise simply set it.
+    // otherwise simply set it.
     $current_process = $this->getProcess();
     if (isset($current_process[$property])) {
       $this->process = NestedArray::mergeDeepArray([$current_process, $this->getProcessNormalized([$property => $process_of_property])], TRUE);

core/modules/migrate/src/Plugin/MigrationInterface.php

@@ -153,7 +153,7 @@ interface MigrationInterface extends PluginInspectionInterface, DerivativeInspec
   /**
    * Set the current migration status.
    *
-   * @param int $result
+   * @param int $status
    *   One of the STATUS_* constants.
    */
   public function setStatus($status);

core/modules/migrate/src/Plugin/MigrationPluginManager.php

@@ -64,7 +64,7 @@ class MigrationPluginManager extends DefaultPluginManager implements MigrationPl
    */
   protected function getDiscovery() {
     if (!isset($this->discovery)) {
-      $directories = array_map(function($directory) {
+      $directories = array_map(function ($directory) {
         return [$directory . '/migration_templates', $directory . '/migrations'];
       }, $this->moduleHandler->getModuleDirectories());
 
@@ -125,7 +125,7 @@ class MigrationPluginManager extends DefaultPluginManager implements MigrationPl
    *   An array of migration objects with the given tag.
    */
   public function createInstancesByTag($tag) {
-    $migrations = array_filter($this->getDefinitions(), function($migration) use ($tag) {
+    $migrations = array_filter($this->getDefinitions(), function ($migration) use ($tag) {
       return !empty($migration['migration_tags']) && in_array($tag, $migration['migration_tags']);
     });
     return $this->createInstances(array_keys($migrations));

core/modules/migrate/src/Plugin/migrate/destination/ComponentEntityDisplayBase.php

@@ -64,6 +64,6 @@ abstract class ComponentEntityDisplayBase extends DestinationBase {
    * @return \Drupal\Core\Entity\Display\EntityDisplayInterface
    *   The entity display object.
    */
-  protected abstract function getEntity($entity_type, $bundle, $mode);
+  abstract protected function getEntity($entity_type, $bundle, $mode);
 
 }

core/modules/migrate/src/Plugin/migrate/destination/Config.php

@@ -195,4 +195,26 @@ class Config extends DestinationBase implements ContainerFactoryPluginInterface,
     }
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function getDestinationModule() {
+    if (!empty($this->configuration['destination_module'])) {
+      return $this->configuration['destination_module'];
+    }
+    if (!empty($this->pluginDefinition['destination_module'])) {
+      return $this->pluginDefinition['destination_module'];
+    }
+    // Config translations require the config_translation module so set the
+    // migration provider to 'config_translation'. The corresponding non
+    // translated configuration is expected to be handled in a separate
+    // migration.
+    if (isset($this->configuration['translations'])) {
+      return 'config_translation';
+    }
+    // Get the module handling this configuration object from the config_name,
+    // which is of the form <module_name>.<configuration object name>
+    return !empty($this->configuration['config_name']) ? explode('.', $this->configuration['config_name'], 2)[0] : NULL;
+  }
+
 }

core/modules/migrate/src/Plugin/migrate/destination/DestinationBase.php

@@ -110,4 +110,22 @@ abstract class DestinationBase extends PluginBase implements MigrateDestinationI
     }
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function getDestinationModule() {
+    if (!empty($this->configuration['destination_module'])) {
+      return $this->configuration['destination_module'];
+    }
+    if (!empty($this->pluginDefinition['destination_module'])) {
+      return $this->pluginDefinition['destination_module'];
+    }
+    if (is_string($this->migration->provider)) {
+      return $this->migration->provider;
+    }
+    else {
+      return reset($this->migration->provider);
+    }
+  }
+
 }

core/modules/migrate/src/Plugin/migrate/destination/EntityContentBase.php

@@ -149,6 +149,7 @@ class EntityContentBase extends Entity {
    *   An updated entity, or NULL if it's the same as the one passed in.
    */
   protected function updateEntity(EntityInterface $entity, Row $row) {
+    $empty_destinations = $row->getEmptyDestinationProperties();
     // By default, an update will be preserved.
     $rollback_action = MigrateIdMapInterface::ROLLBACK_PRESERVE;
 
@@ -171,6 +172,7 @@ class EntityContentBase extends Entity {
     // clone the row with an empty set of destination values, and re-add only
     // the specified properties.
     if (isset($this->configuration['overwrite_properties'])) {
+      $empty_destinations = array_intersect($empty_destinations, $this->configuration['overwrite_properties']);
       $clone = $row->cloneWithoutDestination();
       foreach ($this->configuration['overwrite_properties'] as $property) {
         $clone->setDestinationProperty($property, $row->getDestinationProperty($property));
@@ -184,6 +186,9 @@ class EntityContentBase extends Entity {
         $field->setValue($values);
       }
     }
+    foreach ($empty_destinations as $field_name) {
+      $entity->$field_name = NULL;
+    }
 
     $this->setRollbackAction($row->getIdMap(), $rollback_action);
 

core/modules/migrate/src/Plugin/migrate/id_map/Sql.php

@@ -6,6 +6,7 @@ use Drupal\Component\Utility\Unicode;
 use Drupal\Core\Field\BaseFieldDefinition;
 use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
 use Drupal\Core\Plugin\PluginBase;
+use Drupal\migrate\MigrateMessage;
 use Drupal\migrate\Plugin\MigrationInterface;
 use Drupal\migrate\Event\MigrateIdMapMessageEvent;
 use Drupal\migrate\MigrateException;
@@ -55,7 +56,7 @@ class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryP
   protected $messageTableName;
 
   /**
-   * The migrate message.
+   * The migrate message service.
    *
    * @var \Drupal\migrate\MigrateMessageInterface
    */
@@ -156,6 +157,7 @@ class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryP
     parent::__construct($configuration, $plugin_id, $plugin_definition);
     $this->migration = $migration;
     $this->eventDispatcher = $event_dispatcher;
+    $this->message = new MigrateMessage();
   }
 
   /**

core/modules/migrate/src/Plugin/migrate/process/DedupeBase.php

@@ -17,6 +17,8 @@ Drupal 8.4.x and will be removed before Drupal 9.0.0. Instead, use ' . __NAMESPA
  *
  * @deprecated in Drupal 8.4.x and will be removed in Drupal 9.0.x. Use
  *   \Drupal\migrate\Plugin\migrate\process\MakeUniqueBase instead.
+ *
+ * @see https://www.drupal.org/node/2873762
  */
 abstract class DedupeBase extends MakeUniqueBase {
 }

core/modules/migrate/src/Plugin/migrate/process/DedupeEntity.php

@@ -19,5 +19,7 @@ Drupal 8.4.x and will be removed before Drupal 9.0.0. Instead, use ' . __NAMESPA
  *
  * @deprecated in Drupal 8.4.x and will be removed in Drupal 9.0.x. Use
  *   \Drupal\migrate\Plugin\migrate\process\MakeUniqueEntityField instead.
+ *
+ * @see https://www.drupal.org/node/2873762
  */
-class DedupeEntity extends MakeUniqueEntityField { }
+class DedupeEntity extends MakeUniqueEntityField {}

core/modules/migrate/src/Plugin/migrate/process/FileCopy.php

@@ -19,11 +19,9 @@ use Symfony\Component\DependencyInjection\ContainerInterface;
  * The file can be moved, reused, or set to be automatically renamed if a
  * duplicate exists.
  *
- * The source value is an array of two values:
- * - source: The source path or URI, e.g. '/path/to/foo.txt' or
- *   'public://bar.txt'.
- * - destination: The destination path or URI, e.g. '/path/to/bar.txt' or
- *   'public://foo.txt'.
+ * The source value is an indexed array of two values:
+ * - The source path or URI, e.g. '/path/to/foo.txt' or 'public://bar.txt'.
+ * - The destination URI, e.g. 'public://foo.txt'.
  *
  * Available configuration keys:
  * - move: (optional) Boolean, if TRUE, move the file, otherwise copy the file.
@@ -39,8 +37,9 @@ use Symfony\Component\DependencyInjection\ContainerInterface;
  * process:
  *   path_to_file:
  *     plugin: file_copy
- *     source: /path/to/file.png
- *     destination: /new/path/to/file.png
+ *     source:
+ *       - /path/to/file.png
+ *       - public://new/path/to/file.png
  * @endcode
  *
  * @see \Drupal\migrate\Plugin\MigrateProcessInterface

core/modules/migrate/src/Plugin/migrate/process/Flatten.php

@@ -1,6 +1,7 @@
 <?php
 
 namespace Drupal\migrate\Plugin\migrate\process;
+
 use Drupal\migrate\MigrateExecutableInterface;
 use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\Row;

core/modules/migrate/src/Plugin/migrate/process/Iterator.php

@@ -2,65 +2,20 @@
 
 namespace Drupal\migrate\Plugin\migrate\process;
 
-use Drupal\migrate\ProcessPluginBase;
-use Drupal\migrate\MigrateExecutableInterface;
-use Drupal\migrate\Row;
+@trigger_error('The ' . __NAMESPACE__ . '\Iterator is deprecated in
+Drupal 8.4.x and will be removed before Drupal 9.0.0. Instead, use ' . __NAMESPACE__ . '\SubProcess', E_USER_DEPRECATED);
 
 /**
- * This plugin iterates and processes an array.
+ * Iterates and processes an associative array.
  *
- * @link https://www.drupal.org/node/2135345 Online handbook documentation for iterator process plugin @endlink
+ * @deprecated in Drupal 8.4.x and will be removed in Drupal 9.0.x. Use
+ *   \Drupal\migrate\Plugin\migrate\process\SubProcess instead.
+ *
+ * @see https://www.drupal.org/node/2880427
  *
  * @MigrateProcessPlugin(
  *   id = "iterator",
  *   handle_multiples = TRUE
  * )
  */
-class Iterator extends ProcessPluginBase {
-
-  /**
-   * Runs a process pipeline on each destination property per list item.
-   */
-  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
-    $return = [];
-    if (!is_null($value)) {
-      foreach ($value as $key => $new_value) {
-        $new_row = new Row($new_value, []);
-        $migrate_executable->processRow($new_row, $this->configuration['process']);
-        $destination = $new_row->getDestination();
-        if (array_key_exists('key', $this->configuration)) {
-          $key = $this->transformKey($key, $migrate_executable, $new_row);
-        }
-        $return[$key] = $destination;
-      }
-    }
-    return $return;
-  }
-
-  /**
-   * Runs the process pipeline for the current key.
-   *
-   * @param string|int $key
-   *   The current key.
-   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
-   *   The migrate executable helper class.
-   * @param \Drupal\migrate\Row $row
-   *   The current row after processing.
-   *
-   * @return mixed
-   *   The transformed key.
-   */
-  protected function transformKey($key, MigrateExecutableInterface $migrate_executable, Row $row) {
-    $process = ['key' => $this->configuration['key']];
-    $migrate_executable->processRow($row, $process, $key);
-    return $row->getDestinationProperty('key');
-  }
-
-  /**
-   * {@inheritdoc}
-   */
-  public function multiple() {
-    return TRUE;
-  }
-
-}
+class Iterator extends SubProcess {}

core/modules/migrate/src/Plugin/migrate/process/Log.php

@@ -6,7 +6,6 @@ use Drupal\migrate\MigrateExecutableInterface;
 use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\Row;
 
-
 /**
  * Logs values without changing them.
  *

core/modules/migrate/src/Plugin/migrate/process/Migration.php

@@ -17,4 +17,4 @@ Drupal 8.4.0 and will be removed before Drupal 9.0.0. Instead, use ' . __NAMESPA
  * @deprecated in Drupal 8.3.x and will be removed in Drupal 9.0.x.
  *  Use \Drupal\migrate\Plugin\migrate\process\MigrationLookup instead.
  */
-class Migration extends MigrationLookup { }
+class Migration extends MigrationLookup {}

core/modules/migrate/src/Plugin/migrate/process/MigrationLookup.php

@@ -222,15 +222,16 @@ class MigrationLookup extends ProcessPluginBase implements ContainerFactoryPlugi
       // Do a normal migration with the stub row.
       $migrate_executable->processRow($stub_row, $process);
       $destination_ids = [];
+      $id_map = $migration->getIdMap();
       try {
         $destination_ids = $destination_plugin->import($stub_row);
       }
       catch (\Exception $e) {
-        $migration->getIdMap()->saveMessage($stub_row->getSourceIdValues(), $e->getMessage());
+        $id_map->saveMessage($stub_row->getSourceIdValues(), $e->getMessage());
       }
 
       if ($destination_ids) {
-        $migration->getIdMap()->saveIdMapping($stub_row, $destination_ids, MigrateIdMapInterface::STATUS_NEEDS_UPDATE);
+        $id_map->saveIdMapping($stub_row, $destination_ids, MigrateIdMapInterface::STATUS_NEEDS_UPDATE);
       }
     }
     if ($destination_ids) {

core/modules/migrate/src/Plugin/migrate/process/SubProcess.php

@@ -0,0 +1,166 @@
+<?php
+
+namespace Drupal\migrate\Plugin\migrate\process;
+
+use Drupal\migrate\ProcessPluginBase;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Row;
+
+/**
+ * Runs an array of arrays through its own process pipeline.
+ *
+ * The sub_process plugin accepts an array of associative arrays and runs each
+ * one through its own process pipeline, producing a newly keyed associative
+ * array of transformed values.
+ *
+ * Available configuration keys:
+ *   - process: the plugin(s) that will process each element of the source.
+ *   - key: runs the process pipeline for the key to determine a new dynamic
+ *     name.
+ *
+ * Examples:
+ *
+ * @code
+ * source: Array
+ * (
+ *   [upload] => Array
+ *     (
+ *       [0] => Array
+ *         (
+ *           [fid] => 1
+ *           [list] => 0
+ *           [description] => "File number 1"
+ *         )
+ *       [1] => Array
+ *         (
+ *           [fid] => 2
+ *           [list] => 1
+ *           [description] => "File number 2"
+ *         )
+ *     )
+ * )
+ * ...
+ * @endcode
+ *
+ * The sub_process process plugin will take these arrays one at a time and run
+ * its own process over each one:
+ *
+ * @code
+ * process:
+ *   upload:
+ *     plugin: sub_process
+ *     source: upload
+ *     process:
+ *       target_id:
+ *         plugin: migration_lookup
+ *         migration: d6_file
+ *         source: fid
+ *       display: list
+ *       description: description
+ * @endcode
+ *
+ * In this case, each item in the upload array will be processed by the
+ * sub_process process plugin. The target_id will be found by looking up the
+ * destination value from a previous migration. The display and description
+ * fields will simply be mapped.
+ *
+ * In the next example, normally the array returned from sub_process will have
+ * its original keys. If you need to change the key, it is possible for the
+ * returned array to be keyed by one of the transformed values in the sub-array.
+ *
+ * @code
+ * source: Array
+ * (
+ *   [format] => 1
+ *   [name] => Filtered HTML
+ * ...
+ *   [filters] => Array
+ *     (
+ *       [0] => Array
+ *         (
+ *           [module] => filter
+ *           [delta] => 2
+ *           [weight] => 0
+ *         )
+ *       [1] => Array
+ *         (
+ *           [module] => filter
+ *           [delta] => 0
+ *           [weight] => 1
+ *         )
+ *    )
+ * )
+ * ...
+ *
+ * process:
+ *   filters:
+ *     plugin: iterator
+ *     source: filters
+ *     key: "@id"
+ *     process:
+ *       id:
+ *         plugin: concat
+ *         source:
+ *           - module
+ *           - delta
+ *         delimiter: _
+ * @endcode
+ *
+ * In the above example, the keys of the returned array would be filter_2 and
+ * filter_0
+ *
+ * @see \Drupal\migrate\Plugin\MigrateProcessInterface
+ *
+ * @MigrateProcessPlugin(
+ *   id = "sub_process",
+ *   handle_multiples = TRUE
+ * )
+ */
+class SubProcess extends ProcessPluginBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
+    $return = [];
+    if (is_array($value) || $value instanceof \Traversable) {
+      foreach ($value as $key => $new_value) {
+        $new_row = new Row($new_value, []);
+        $migrate_executable->processRow($new_row, $this->configuration['process']);
+        $destination = $new_row->getDestination();
+        if (array_key_exists('key', $this->configuration)) {
+          $key = $this->transformKey($key, $migrate_executable, $new_row);
+        }
+        $return[$key] = $destination;
+      }
+    }
+    return $return;
+  }
+
+  /**
+   * Runs the process pipeline for the key to determine its dynamic name.
+   *
+   * @param string|int $key
+   *   The current key.
+   * @param \Drupal\migrate\MigrateExecutableInterface $migrate_executable
+   *   The migrate executable helper class.
+   * @param \Drupal\migrate\Row $row
+   *   The current row after processing.
+   *
+   * @return mixed
+   *   The transformed key.
+   */
+  protected function transformKey($key, MigrateExecutableInterface $migrate_executable, Row $row) {
+    $process = ['key' => $this->configuration['key']];
+    $migrate_executable->processRow($row, $process, $key);
+    return $row->getDestinationProperty('key');
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function multiple() {
+    return TRUE;
+  }
+
+}

core/modules/migrate/src/Plugin/migrate/source/EmbeddedDataSource.php

@@ -28,8 +28,8 @@ use Drupal\migrate\Plugin\MigrationInterface;
  *     -
  *       channel_machine_name: movies
  *       channel_description: Movies
- *     ids:
- *       channel_machine_name:
+ *   ids:
+ *     channel_machine_name:
  *       type: string
  * @endcode
  *

core/modules/migrate/src/Plugin/migrate/source/SourcePluginBase.php

@@ -13,7 +13,49 @@ use Drupal\migrate\Plugin\MigrateSourceInterface;
 use Drupal\migrate\Row;
 
 /**
- * The base class for all source plugins.
+ * The base class for source plugins.
+ *
+ * Available configuration keys:
+ * - cache_counts: (optional) If set, cache the source count.
+ * - skip_count: (optional) If set, do not attempt to count the source.
+ * - track_changes: (optional) If set, track changes to incoming data.
+ * - high_water_property: (optional) It is an array of name & alias values
+ *   (optional table alias). This high_water_property is typically a timestamp
+ *   or serial id showing what was the last imported record. Only content with a
+ *   higher value will be imported.
+ *
+ * The high_water_property and track_changes are mutually exclusive.
+ *
+ * Example:
+ *
+ * @code
+ * source:
+ *   plugin: some_source_plugin_name
+ *   cache_counts: true
+ *   track_changes: true
+ * @endcode
+ *
+ * This example uses the plugin "some_source_plugin_name" and caches the count
+ * of available source records to save calculating it every time count() is
+ * called. Changes to incoming data are watched (because track_changes is true),
+ * which can affect the result of prepareRow().
+ *
+ * Example:
+ *
+ * @code
+ * source:
+ *   plugin: some_source_plugin_name
+ *   skip_count: true
+ *   high_water_property:
+ *     name: changed
+ *     alias: n
+ * @endcode
+ *
+ * In this example, skip_count is true which means count() will not attempt to
+ * count the available source records, but just always return -1 instead. The
+ * high_water_property defines which field marks the last imported row of the
+ * migration. This will get converted into a SQL condition that looks like
+ * 'n.changed' or 'changed' if no alias.
  *
  * @see \Drupal\migrate\Plugin\MigratePluginManager
  * @see \Drupal\migrate\Annotation\MigrateSource
@@ -177,7 +219,7 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
    * @return array
    *   An array of the data for this source.
    */
-  protected abstract function initializeIterator();
+  abstract protected function initializeIterator();
 
   /**
    * Gets the module handler.
@@ -541,4 +583,17 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
     $this->saveHighWater(NULL);
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function getSourceModule() {
+    if (!empty($this->configuration['source_module'])) {
+      return $this->configuration['source_module'];
+    }
+    elseif (!empty($this->pluginDefinition['source_module'])) {
+      return $this->pluginDefinition['source_module'];
+    }
+    return NULL;
+  }
+
 }

core/modules/migrate/src/Plugin/migrate/source/SqlBase.php

@@ -15,13 +15,37 @@ use Drupal\migrate\Plugin\RequirementsInterface;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
- * Sources whose data may be fetched via DBTNG.
+ * Sources whose data may be fetched via a database connection.
  *
- * By default, an existing database connection with key 'migrate' and target
- * 'default' is used. These may be overridden with explicit 'key' and/or
- * 'target' configuration keys. In addition, if the configuration key 'database'
- * is present, it is used as a database connection information array to define
- * the connection.
+ * Database configuration, which may appear either within the source plugin
+ * configuration or in state, is structured as follows:
+ *
+ * 'key' - The database key name (defaults to 'migrate').
+ * 'target' - The database target name (defaults to 'default').
+ * 'database' - Database connection information as accepted by
+ *   Database::addConnectionInfo(). If not present, the key/target is assumed
+ *   to already be defined (e.g., in settings.php).
+ *
+ * This configuration info is obtained in the following order:
+ *
+ * 1. If the source plugin configuration contains a key 'database_state_key',
+ *    its value is taken as the name of a state key which contains an array
+ *    with the above database configuration.
+ * 2. Otherwise, if the source plugin configuration contains 'key', the above
+ *    database configuration is obtained directly from the plugin configuration.
+ * 3. Otherwise, if the state 'migrate.fallback_state_key' exists, its value is
+ *    taken as the name of a state key which contains an array with the above
+ *    database configuration.
+ * 4. Otherwise, if a connection named 'migrate' exists, that is used as the
+ *    database connection.
+ * 5. Otherwise, RequirementsException is thrown.
+ *
+ * It is strongly recommended that database connections be explicitly defined
+ * via 'database_state_key' or in the source plugin configuration. Defining
+ * migrate.fallback_state_key or a 'migrate' connection affects not only any
+ * migrations intended to use that particular connection, but all
+ * SqlBase-derived source plugins which do not have explicit database
+ * configuration.
  */
 abstract class SqlBase extends SourcePluginBase implements ContainerFactoryPluginInterface, RequirementsInterface {
 
@@ -101,16 +125,21 @@ abstract class SqlBase extends SourcePluginBase implements ContainerFactoryPlugi
    */
   public function getDatabase() {
     if (!isset($this->database)) {
-      // See if the database info is in state - if not, fallback to
-      // configuration.
+      // Look first for an explicit state key containing the configuration.
       if (isset($this->configuration['database_state_key'])) {
         $this->database = $this->setUpDatabase($this->state->get($this->configuration['database_state_key']));
       }
+      // Next, use explicit configuration in the source plugin.
+      elseif (isset($this->configuration['key'])) {
+        $this->database = $this->setUpDatabase($this->configuration);
+      }
+      // Next, try falling back to the global state key.
       elseif (($fallback_state_key = $this->state->get('migrate.fallback_state_key'))) {
         $this->database = $this->setUpDatabase($this->state->get($fallback_state_key));
       }
+      // If all else fails, let setUpDatabase() fallback to the 'migrate' key.
       else {
-        $this->database = $this->setUpDatabase($this->configuration);
+        $this->database = $this->setUpDatabase([]);
       }
     }
     return $this->database;
@@ -274,11 +303,17 @@ abstract class SqlBase extends SourcePluginBase implements ContainerFactoryPlugi
       }
       // 2. If we are using high water marks, also include rows above the mark.
       //    But, include all rows if the high water mark is not set.
-      if ($this->getHighWaterProperty() && ($high_water = $this->getHighWater())) {
+      if ($this->getHighWaterProperty()) {
         $high_water_field = $this->getHighWaterField();
-        $conditions->condition($high_water_field, $high_water, '>');
+        $high_water = $this->getHighWater();
+        if ($high_water) {
+          $conditions->condition($high_water_field, $high_water, '>');
+          $condition_added = TRUE;
+        }
+        // Always sort by the high water field, to ensure that the first run
+        // (before we have a high water value) also has the results in a
+        // consistent order.
         $this->query->orderBy($high_water_field);
-        $condition_added = TRUE;
       }
       if ($condition_added) {
         $this->query->condition($conditions);

core/modules/migrate/src/Row.php

@@ -77,6 +77,13 @@ class Row {
    */
   protected $isStub = FALSE;
 
+  /**
+   * The empty destination properties.
+   *
+   * @var array
+   */
+  protected $emptyDestinationProperties = [];
+
   /**
    * Constructs a \Drupal\Migrate\Row object.
    *
@@ -229,6 +236,26 @@ class Row {
     NestedArray::unsetValue($this->destination, explode(static::PROPERTY_SEPARATOR, $property));
   }
 
+  /**
+   * Sets a destination to be empty.
+   *
+   * @param string $property
+   *   The destination property.
+   */
+  public function setEmptyDestinationProperty($property) {
+    $this->emptyDestinationProperties[] = $property;
+  }
+
+  /**
+   * Gets the empty destination properties.
+   *
+   * @return array
+   *   An array of destination properties.
+   */
+  public function getEmptyDestinationProperties() {
+    return $this->emptyDestinationProperties;
+  }
+
   /**
    * Returns the whole destination array.
    *