123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388 |
- <?php
- namespace Drupal\simple_sitemap\Queue;
- use Drupal\Component\Utility\Timer;
- use Drupal\simple_sitemap\Plugin\simple_sitemap\SitemapGenerator\SitemapGeneratorBase;
- use Drupal\simple_sitemap\SimplesitemapSettings;
- use Drupal\simple_sitemap\SimplesitemapManager;
- use Drupal\Core\State\StateInterface;
- use Drupal\simple_sitemap\Logger;
- class QueueWorker {
- use BatchTrait;
- const REBUILD_QUEUE_CHUNK_ITEM_SIZE = 5000;
- /**
- * @var \Drupal\simple_sitemap\SimplesitemapSettings
- */
- protected $settings;
- /**
- * @var \Drupal\simple_sitemap\SimplesitemapManager
- */
- protected $manager;
- /**
- * @var \Drupal\Core\State\StateInterface
- */
- protected $state;
- /**
- * @var \Drupal\simple_sitemap\Queue\SimplesitemapQueue
- */
- protected $queue;
- /**
- * @var \Drupal\simple_sitemap\Logger
- */
- protected $logger;
- /**
- * @var string|null
- */
- protected $variantProcessedNow;
- /**
- * @var string|null
- */
- protected $generatorProcessedNow;
- /**
- * @var array
- */
- protected $results = [];
- /**
- * @var array
- */
- protected $processedPaths = [];
- /**
- * @var array
- */
- protected $generatorSettings;
- /**
- * @var int|null
- */
- protected $maxLinks;
- /**
- * @var int|null
- */
- protected $elementsRemaining;
- /**
- * @var int|null
- */
- protected $elementsTotal;
- /**
- * QueueWorker constructor.
- * @param \Drupal\simple_sitemap\SimplesitemapSettings $settings
- * @param \Drupal\simple_sitemap\SimplesitemapManager $manager
- * @param \Drupal\Core\State\StateInterface $state
- * @param \Drupal\simple_sitemap\Queue\SimplesitemapQueue $element_queue
- * @param \Drupal\simple_sitemap\Logger $logger
- */
- public function __construct(SimplesitemapSettings $settings,
- SimplesitemapManager $manager,
- StateInterface $state,
- SimplesitemapQueue $element_queue,
- Logger $logger) {
- $this->settings = $settings;
- $this->manager = $manager;
- $this->state = $state;
- $this->queue = $element_queue;
- $this->logger = $logger;
- }
- /**
- * @return $this
- */
- public function deleteQueue() {
- $this->queue->deleteQueue();
- SitemapGeneratorBase::purgeSitemapVariants(NULL, 'unpublished');
- $this->variantProcessedNow = NULL;
- $this->generatorProcessedNow = NULL;
- $this->results = [];
- $this->processedPaths = [];
- $this->state->set('simple_sitemap.queue_items_initial_amount', 0);
- $this->state->delete('simple_sitemap.queue_stashed_results');
- $this->elementsTotal = NULL;
- $this->elementsRemaining = NULL;
- return $this;
- }
- /**
- * @param array|null $variants
- * @return $this
- * @throws \Drupal\Component\Plugin\Exception\PluginException
- */
- public function rebuildQueue($variants = NULL) {
- $all_data_sets = [];
- $sitemap_variants = $this->manager->getSitemapVariants();
- $type_definitions = $this->manager->getSitemapTypes();
- $this->deleteQueue();
- foreach ($sitemap_variants as $variant_name => $variant_definition) {
- // Skipping unwanted sitemap variants.
- if (NULL !== $variants && !in_array($variant_name, (array) $variants)) {
- continue;
- }
- $type = $variant_definition['type'];
- // Adding generate_sitemap operations for all data sets.
- foreach ($type_definitions[$type]['urlGenerators'] as $url_generator_id) {
- $data_sets = $this->manager->getUrlGenerator($url_generator_id)
- ->setSitemapVariant($variant_name)
- ->getDataSets();
- if (!empty($data_sets)) {
- $sitemap_variants[$variant_name]['data'] = TRUE;
- foreach ($data_sets as $data_set) {
- $all_data_sets[] = [
- 'data' => $data_set,
- 'sitemap_variant' => $variant_name,
- 'url_generator' => $url_generator_id,
- 'sitemap_generator' => $type_definitions[$type]['sitemapGenerator'],
- ];
- if (count($all_data_sets) === self::REBUILD_QUEUE_CHUNK_ITEM_SIZE) {
- $this->queueElements($all_data_sets);
- $all_data_sets = [];
- }
- }
- }
- }
- }
- if (!empty($all_data_sets)) {
- $this->queueElements($all_data_sets);
- }
- $this->getQueuedElementCount(TRUE);
- // todo: May not be clean to remove sitemap variants data when queuing elements.
- // todo: Add test.
- // Remove all sitemap variant instances where no results have been queued.
- $this->manager->removeSitemap(array_keys(array_filter($sitemap_variants, function($e) { return empty($e['data']); })));
- return $this;
- }
- protected function queueElements($elements) {
- $this->queue->createItems($elements);
- $this->state->set('simple_sitemap.queue_items_initial_amount', ($this->state->get('simple_sitemap.queue_items_initial_amount') + count($elements)));
- }
- /**
- * @param string $from
- * @return $this
- * @throws \Drupal\Component\Plugin\Exception\PluginException
- */
- public function generateSitemap($from = 'form') {
- $this->generatorSettings = [
- 'base_url' => $this->settings->getSetting('base_url', ''),
- 'default_variant' => $this->settings->getSetting('default_variant', NULL),
- 'skip_untranslated' => $this->settings->getSetting('skip_untranslated', FALSE),
- 'remove_duplicates' => $this->settings->getSetting('remove_duplicates', TRUE),
- 'excluded_languages' => $this->settings->getSetting('excluded_languages', []),
- ];
- $this->maxLinks = $this->settings->getSetting('max_links');
- $max_execution_time = $this->settings->getSetting('generate_duration', 10000);
- Timer::start('simple_sitemap_generator');
- $this->unstashResults();
- if (!$this->generationInProgress()) {
- $this->rebuildQueue();
- }
- while ($element = $this->queue->claimItem()) {
- if (!empty($max_execution_time) && Timer::read('simple_sitemap_generator') >= $max_execution_time) {
- break;
- }
- try {
- if ($element->data['sitemap_variant'] !== $this->variantProcessedNow) {
- if (NULL !== $this->variantProcessedNow) {
- $this->generateVariantChunksFromResults(TRUE);
- $this->publishCurrentVariant();
- }
- $this->variantProcessedNow = $element->data['sitemap_variant'];
- $this->generatorProcessedNow = $element->data['sitemap_generator'];
- $this->processedPaths = [];
- }
- $this->generateResultsFromElement($element);
- if (!empty($this->maxLinks) && count($this->results) >= $this->maxLinks) {
- $this->generateVariantChunksFromResults();
- }
- }
- catch (\Exception $e) {
- watchdog_exception('simple_sitemap', $e);
- }
- $this->queue->deleteItem($element); //todo May want to use deleteItems() instead.
- $this->elementsRemaining--;
- }
- if ($this->getQueuedElementCount() === 0) {
- $this->generateVariantChunksFromResults(TRUE);
- $this->publishCurrentVariant();
- }
- else {
- $this->stashResults();
- }
- return $this;
- }
- /**
- * @param $element
- * @throws \Drupal\Component\Plugin\Exception\PluginException
- */
- protected function generateResultsFromElement($element) {
- $results = $this->manager->getUrlGenerator($element->data['url_generator'])
- ->setSitemapVariant($this->variantProcessedNow)
- ->setSettings($this->generatorSettings)
- ->generate($element->data['data']);
- $this->removeDuplicates($results);
- $this->results = array_merge($this->results, $results);
- }
- /**
- * @param array $results
- */
- protected function removeDuplicates(&$results) {
- if ($this->generatorSettings['remove_duplicates'] && !empty($results)) {
- $result = $results[key($results)];
- if (!empty($result['meta']['path'])) {
- if (in_array($result['meta']['path'], $this->processedPaths)) {
- $results = [];
- }
- else {
- $this->processedPaths[] = $result['meta']['path'];
- }
- }
- }
- }
- /**
- * @param bool $complete
- * @throws \Drupal\Component\Plugin\Exception\PluginException
- */
- protected function generateVariantChunksFromResults($complete = FALSE) {
- if (!empty($this->results)) {
- $generator = $this->manager->getSitemapGenerator($this->generatorProcessedNow)
- ->setSitemapVariant($this->variantProcessedNow)
- ->setSettings($this->generatorSettings);
- if (empty($this->maxLinks) || $complete) {
- $generator->generate($this->results);
- $this->results = [];
- }
- else {
- foreach (array_chunk($this->results, $this->maxLinks, TRUE) as $chunk_links) {
- if (count($chunk_links) === $this->maxLinks || $complete) {
- $generator->generate($chunk_links);
- $this->results = array_diff_key($this->results, $chunk_links);
- }
- }
- }
- };
- }
- protected function publishCurrentVariant() {
- if ($this->variantProcessedNow !== NULL) {
- $this->manager->getSitemapGenerator($this->generatorProcessedNow)
- ->setSitemapVariant($this->variantProcessedNow)
- ->setSettings($this->generatorSettings)
- ->generateIndex()
- ->publish();
- }
- }
- protected function stashResults() {
- $this->state->set('simple_sitemap.queue_stashed_results', [
- 'variant' => $this->variantProcessedNow,
- 'generator' => $this->generatorProcessedNow,
- 'results' => $this->results,
- 'processed_paths' => $this->processedPaths,
- ]);
- $this->results = [];
- $this->processedPaths = [];
- $this->generatorProcessedNow = NULL;
- $this->variantProcessedNow = NULL;
- }
- protected function unstashResults() {
- if (NULL !== $results = $this->state->get('simple_sitemap.queue_stashed_results')) {
- $this->state->delete('simple_sitemap.queue_stashed_results');
- $this->results = !empty($results['results']) ? $results['results'] : [];
- $this->processedPaths = !empty($results['processed_paths']) ? $results['processed_paths'] : [];
- $this->variantProcessedNow = $results['variant'];
- $this->generatorProcessedNow = $results['generator'];
- }
- }
- public function getInitialElementCount() {
- if (NULL === $this->elementsTotal) {
- $this->elementsTotal = (int) $this->state->get('simple_sitemap.queue_items_initial_amount', 0);
- }
- return $this->elementsTotal;
- }
- /**
- * @param bool $force_recount
- * @return int
- */
- public function getQueuedElementCount($force_recount = FALSE) {
- if ($force_recount || NULL === $this->elementsRemaining) {
- $this->elementsRemaining = $this->queue->numberOfItems();
- }
- return $this->elementsRemaining;
- }
- /**
- * @return int
- */
- public function getStashedResultCount() {
- return count($this->state->get('simple_sitemap.queue_stashed_results', ['results' => []])['results']);
- }
- /**
- * @return int
- */
- public function getProcessedElementCount() {
- $initial = $this->getInitialElementCount();
- $remaining = $this->getQueuedElementCount();
- return $initial > $remaining ? ($initial - $remaining) : 0;
- }
- /**
- * @return bool
- */
- public function generationInProgress() {
- return 0 < ($this->getQueuedElementCount() + $this->getStashedResultCount());
- }
- }
|