added redis module

This commit is contained in:
Bachir Soussi Chiadmi
2018-01-23 21:13:49 +01:00
parent fdb2089ae1
commit 10c7c59e05
44 changed files with 4225 additions and 0 deletions
@@ -0,0 +1,70 @@
<?php
namespace Drupal\redis\Cache;
use Drupal\Component\Serialization\SerializationInterface;
use Drupal\Core\Cache\CacheFactoryInterface;
use Drupal\Core\Cache\CacheTagsChecksumInterface;
use Drupal\redis\ClientFactory;
/**
* A cache backend factory responsible for the construction of redis cache bins.
*/
class CacheBackendFactory implements CacheFactoryInterface {
/**
* @var \Drupal\redis\ClientInterface
*/
protected $clientFactory;
/**
* The cache tags checksum provider.
*
* @var \Drupal\Core\Cache\CacheTagsChecksumInterface
*/
protected $checksumProvider;
/**
* The serialization class to use.
*
* @var \Drupal\Component\Serialization\SerializationInterface
*/
protected $serializer;
/**
* List of cache bins.
*
* Renderer and possibly other places fetch backends directly from the
* factory. Avoid that the backend objects have to fetch meta information like
* the last delete all timestamp multiple times.
*
* @var array
*/
protected $bins = [];
/**
* Creates a redis CacheBackendFactory.
*
* @param \Drupal\redis\ClientFactory $client_factory
* @param \Drupal\Core\Cache\CacheTagsChecksumInterface $checksum_provider
* @param \Drupal\redis\Cache\SerializationInterface $serializer
* The serialization class to use.
*/
public function __construct(ClientFactory $client_factory, CacheTagsChecksumInterface $checksum_provider, SerializationInterface $serializer) {
$this->clientFactory = $client_factory;
$this->checksumProvider = $checksum_provider;
$this->serializer = $serializer;
}
/**
* {@inheritdoc}
*/
public function get($bin) {
if (!isset($this->bins[$bin])) {
$class_name = $this->clientFactory->getClass(ClientFactory::REDIS_IMPL_CACHE);
$this->bins[$bin] = new $class_name($bin, $this->clientFactory->getClient(), $this->checksumProvider, $this->serializer);
}
return $this->bins[$bin];
}
}
@@ -0,0 +1,378 @@
<?php
namespace Drupal\redis\Cache;
use \DateInterval;
use Drupal\Component\Serialization\SerializationInterface;
use Drupal\Core\Cache\Cache;
use Drupal\Core\Cache\CacheBackendInterface;
use Drupal\Core\Site\Settings;
use Drupal\redis\RedisPrefixTrait;
/**
* Base class for redis cache backends.
*
* *
*
*/
abstract class CacheBase implements CacheBackendInterface {
use RedisPrefixTrait;
/**
* Temporary cache items lifetime is infinite.
*/
const LIFETIME_INFINITE = 0;
/**
* Default lifetime for permanent items.
* Approximatively 1 year.
*/
const LIFETIME_PERM_DEFAULT = 31536000;
/**
* Computed keys are let's say arround 60 characters length due to
* key prefixing, which makes 1,000 keys DEL command to be something
* arround 50,000 bytes length: this is huge and may not pass into
* Redis, let's split this off.
* Some recommend to never get higher than 1,500 bytes within the same
* command which makes us forced to split this at a very low threshold:
* 20 seems a safe value here (1,280 average length).
*/
const KEY_THRESHOLD = 20;
/**
* Latest delete all flush KEY name.
*/
const LAST_DELETE_ALL_KEY = '_redis_last_delete_all';
/**
* @var string
*/
protected $bin;
/**
* The serialization class to use.
*
* @var \Drupal\Component\Serialization\SerializationInterface
*/
protected $serializer;
/**
* Default TTL for CACHE_PERMANENT items.
*
* See "Default lifetime for permanent items" section of README.md
* file for a comprehensive explaination of why this exists.
*
* @var int
*/
protected $permTtl = self::LIFETIME_PERM_DEFAULT;
/**
* Minimal TTL to use.
*
* Note that this is for testing purposes. Do not specify the minimal TTL
* outside of unit-tests.
*/
protected $minTtl = 0;
/**
* @var \Drupal\redis\ClientInterface
*/
protected $client;
/**
* The cache tags checksum provider.
*
* @var \Drupal\Core\Cache\CacheTagsChecksumInterface|\Drupal\Core\Cache\CacheTagsInvalidatorInterface
*/
protected $checksumProvider;
/**
* The last delete timestamp.
*
* @var float
*/
protected $lastDeleteAll = NULL;
/**
* Get TTL for CACHE_PERMANENT items.
*
* @return int
* Lifetime in seconds.
*/
public function getPermTtl() {
return $this->permTtl;
}
/**
* CacheBase constructor.
* @param $bin
* The cache bin for which the object is created.
* @param \Drupal\Component\Serialization\SerializationInterface $serializer
* The serialization class to use.
*/
public function __construct($bin, SerializationInterface $serializer) {
$this->bin = $bin;
$this->serializer = $serializer;
$this->setPermTtl();
}
/**
* {@inheritdoc}
*/
public function get($cid, $allow_invalid = FALSE) {
$cids = [$cid];
$cache = $this->getMultiple($cids, $allow_invalid);
return reset($cache);
}
/**
* {@inheritdoc}
*/
public function setMultiple(array $items) {
foreach ($items as $cid => $item) {
$this->set($cid, $item['data'], isset($item['expire']) ? $item['expire'] : CacheBackendInterface::CACHE_PERMANENT, isset($item['tags']) ? $item['tags'] : []);
}
}
/**
* {@inheritdoc}
*/
public function delete($cid) {
$this->deleteMultiple([$cid]);
}
/**
* {@inheritdoc}
*/
public function removeBin() {
$this->deleteAll();
}
/**
* {@inheritdoc}
*/
public function invalidate($cid) {
$this->invalidateMultiple([$cid]);
}
/**
* Return the key for the given cache key.
*/
public function getKey($cid = NULL) {
if (NULL === $cid) {
return $this->getPrefix() . ':' . $this->bin;
}
else {
return $this->getPrefix() . ':' . $this->bin . ':' . $cid;
}
}
/**
* Calculate the correct expiration time.
*
* @param int $expire
* The expiration time provided for the cache set.
*
* @return int
* The default expiration if expire is PERMANENT or higher than the default.
* May return negative values if the item is already expired.
*/
protected function getExpiration($expire) {
if ($expire == Cache::PERMANENT || $expire > $this->permTtl) {
return $this->permTtl;
}
return $expire - REQUEST_TIME;
}
/**
* Return the key for the tag used to specify the bin of cache-entries.
*/
protected function getTagForBin() {
return 'x-redis-bin:' . $this->bin;
}
/**
* Set the minimum TTL (unit testing only).
*/
public function setMinTtl($ttl) {
$this->minTtl = $ttl;
}
/**
* Set the permanent TTL.
*/
public function setPermTtl($ttl = NULL) {
if (isset($ttl)) {
$this->permTtl = $ttl;
}
else {
// Attempt to set from settings.
if (($settings = Settings::get('redis.settings', [])) && isset($settings['perm_ttl_' . $this->bin])) {
$ttl = $settings['perm_ttl_' . $this->bin];
if ($ttl === (int) $ttl) {
$this->permTtl = $ttl;
}
else {
if ($iv = DateInterval::createFromDateString($ttl)) {
// http://stackoverflow.com/questions/14277611/convert-dateinterval-object-to-seconds-in-php
$this->permTtl = ($iv->y * 31536000 + $iv->m * 2592000 + $iv->days * 86400 + $iv->h * 3600 + $iv->i * 60 + $iv->s);
}
else {
// Log error about invalid ttl.
trigger_error(sprintf("Parsed TTL '%s' has an invalid value: switching to default", $ttl));
$this->permTtl = self::LIFETIME_PERM_DEFAULT;
}
}
}
}
}
/**
* Prepares a cached item.
*
* Checks that items are either permanent or did not expire, and unserializes
* data as appropriate.
*
* @param array $values
* The hash returned from redis or false.
* @param bool $allow_invalid
* If FALSE, the method returns FALSE if the cache item is not valid.
*
* @return mixed|false
* The item with data unserialized as appropriate and a property indicating
* whether the item is valid, or FALSE if there is no valid item to load.
*/
protected function expandEntry(array $values, $allow_invalid) {
// Check for entry being valid.
if (empty($values['cid'])) {
return FALSE;
}
$cache = (object) $values;
$cache->tags = explode(' ', $cache->tags);
// Check expire time, allow to have a cache invalidated explicitly, don't
// check if already invalid.
if ($cache->valid) {
$cache->valid = $cache->expire == Cache::PERMANENT || $cache->expire >= REQUEST_TIME;
// Check if invalidateTags() has been called with any of the items's tags.
if ($cache->valid && !$this->checksumProvider->isValid($cache->checksum, $cache->tags)) {
$cache->valid = FALSE;
}
}
// Ensure the entry does not predate the last delete all time.
$last_delete_timestamp = $this->getLastDeleteAll();
if ($last_delete_timestamp && ((float)$values['created']) < $last_delete_timestamp) {
return FALSE;
}
if (!$allow_invalid && !$cache->valid) {
return FALSE;
}
if ($cache->serialized) {
$cache->data = $this->serializer->decode($cache->data);
}
return $cache;
}
/**
* Create cache entry.
*
* @param string $cid
* @param mixed $data
* @param int $expire
* @param string[] $tags
*
* @return array
*/
protected function createEntryHash($cid, $data, $expire = Cache::PERMANENT, array $tags) {
// Always add a cache tag for the current bin, so that we can use that for
// invalidateAll().
$tags[] = $this->getTagForBin();
assert('\Drupal\Component\Assertion\Inspector::assertAllStrings($tags)', 'Cache Tags must be strings.');
$hash = [
'cid' => $cid,
'created' => round(microtime(TRUE), 3),
'expire' => $expire,
'tags' => implode(' ', $tags),
'valid' => 1,
'checksum' => $this->checksumProvider->getCurrentChecksum($tags),
];
// Let Redis handle the data types itself.
if (!is_string($data)) {
$hash['data'] = $this->serializer->encode($data);
$hash['serialized'] = 1;
}
else {
$hash['data'] = $data;
$hash['serialized'] = 0;
}
return $hash;
}
/**
* {@inheritdoc}
*/
public function invalidateMultiple(array $cids) {
// Loop over all cache items, they are stored as a hash, so we can access
// the valid flag directly, only write if it exists and is not 0.
foreach ($cids as $cid) {
$key = $this->getKey($cid);
if ($this->client->hGet($key, 'valid')) {
$this->client->hSet($key, 'valid', 0);
}
}
}
/**
* {@inheritdoc}
*/
public function invalidateAll() {
// To invalidate the whole bin, we invalidate a special tag for this bin.
$this->checksumProvider->invalidateTags([$this->getTagForBin()]);
}
/**
* {@inheritdoc}
*/
public function garbageCollection() {
// @todo Do we need to do anything here?
}
/**
* Returns the last delete all timestamp.
*
* @return float
* The last delete timestamp as a timestamp with a millisecond precision.
*/
protected function getLastDeleteAll() {
// Cache the last delete all timestamp.
if ($this->lastDeleteAll === NULL) {
$this->lastDeleteAll = (float) $this->client->get($this->getKey(static::LAST_DELETE_ALL_KEY));
}
return $this->lastDeleteAll;
}
/**
* {@inheritdoc}
*/
public function deleteAll() {
// The last delete timestamp is in milliseconds, ensure that no cache
// was written in the same millisecond.
// @todo This is needed to make the tests pass, is this safe enough for real
// usage?
usleep(1000);
$this->lastDeleteAll = round(microtime(TRUE), 3);
$this->client->set($this->getKey(static::LAST_DELETE_ALL_KEY), $this->lastDeleteAll);
}
}
@@ -0,0 +1,111 @@
<?php
namespace Drupal\redis\Cache;
use Drupal\Component\Serialization\SerializationInterface;
use Drupal\Core\Cache\Cache;
use Drupal\Core\Cache\CacheTagsChecksumInterface;
/**
* PhpRedis cache backend.
*/
class PhpRedis extends CacheBase {
/**
* @var \Redis
*/
protected $client;
/**
* Creates a PHpRedis cache backend.
*
* @param $bin
* The cache bin for which the object is created.
* @param \Redis $client
* @param \Drupal\Core\Cache\CacheTagsChecksumInterface $checksum_provider
* @param \Drupal\redis\Cache\SerializationInterface $serializer
* The serialization class to use.
*/
public function __construct($bin, \Redis $client, CacheTagsChecksumInterface $checksum_provider, SerializationInterface $serializer) {
parent::__construct($bin, $serializer);
$this->client = $client;
$this->checksumProvider = $checksum_provider;
}
/**
* {@inheritdoc}
*/
public function getMultiple(&$cids, $allow_invalid = FALSE) {
// Avoid an error when there are no cache ids.
if (empty($cids)) {
return [];
}
$return = [];
// Build the list of keys to fetch.
$keys = array_map([$this, 'getKey'], $cids);
// Optimize for the common case when only a single cache entry needs to
// be fetched, no pipeline is needed then.
if (count($keys) > 1) {
$pipe = $this->client->multi(\Redis::PIPELINE);
foreach ($keys as $key) {
$pipe->hgetall($key);
}
$result = $pipe->exec();
}
else {
$result = [$this->client->hGetAll(reset($keys))];
}
// Loop over the cid values to ensure numeric indexes.
foreach (array_values($cids) as $index => $key) {
// Check if a valid result was returned from Redis.
if (isset($result[$index]) && is_array($result[$index])) {
// Check expiration and invalidation and convert into an object.
$item = $this->expandEntry($result[$index], $allow_invalid);
if ($item) {
$return[$item->cid] = $item;
}
}
}
// Remove fetched cids from the list.
$cids = array_diff($cids, array_keys($return));
return $return;
}
/**
* {@inheritdoc}
*/
public function set($cid, $data, $expire = Cache::PERMANENT, array $tags = []) {
$ttl = $this->getExpiration($expire);
$key = $this->getKey($cid);
// If the item is already expired, delete it.
if ($ttl <= 0) {
$this->delete($key);
}
// Build the cache item and save it as a hash array.
$entry = $this->createEntryHash($cid, $data, $expire, $tags);
$pipe = $this->client->multi(\REdis::PIPELINE);
$pipe->hMset($key, $entry);
$pipe->expire($key, $ttl);
$pipe->exec();
}
/**
* {@inheritdoc}
*/
public function deleteMultiple(array $cids) {
$keys = array_map([$this, 'getKey'], $cids);
$this->client->del($keys);
}
}
@@ -0,0 +1,113 @@
<?php
namespace Drupal\redis\Cache;
use Drupal\Component\Serialization\SerializationInterface;
use Drupal\Core\Cache\Cache;
use Drupal\Core\Cache\CacheTagsChecksumInterface;
/**
* Predis cache backend.
*/
class Predis extends CacheBase {
/**
* @var \Predis\Client
*/
protected $client;
/**
* Creates a Predis cache backend.
*
* @param $bin
* The cache bin for which the object is created.
* @param \Redis $client
* @param \Drupal\Core\Cache\CacheTagsChecksumInterface $checksum_provider
* @param \Drupal\redis\Cache\SerializationInterface $serializer
* The serialization class to use.
*/
public function __construct($bin, \Predis\Client $client, CacheTagsChecksumInterface $checksum_provider, SerializationInterface $serializer) {
parent::__construct($bin, $serializer);
$this->client = $client;
$this->checksumProvider = $checksum_provider;
}
/**
* {@inheritdoc}
*/
public function getMultiple(&$cids, $allow_invalid = FALSE) {
// Avoid an error when there are no cache ids.
if (empty($cids)) {
return [];
}
$return = [];
// Build the list of keys to fetch.
$keys = array_map([$this, 'getKey'], $cids);
// Optimize for the common case when only a single cache entry needs to
// be fetched, no pipeline is needed then.
if (count($keys) > 1) {
$pipe = $this->client->pipeline();
foreach ($keys as $key) {
$pipe->hgetall($key);
}
$result = $pipe->execute();
}
else {
$result = [$this->client->hGetAll(reset($keys))];
}
// Loop over the cid values to ensure numeric indexes.
foreach (array_values($cids) as $index => $key) {
// Check if a valid result was returned from Redis.
if (isset($result[$index]) && is_array($result[$index])) {
// Check expiration and invalidation and convert into an object.
$item = $this->expandEntry($result[$index], $allow_invalid);
if ($item) {
$return[$item->cid] = $item;
}
}
}
// Remove fetched cids from the list.
$cids = array_diff($cids, array_keys($return));
return $return;
}
/**
* {@inheritdoc}
*/
public function set($cid, $data, $expire = Cache::PERMANENT, array $tags = []) {
$ttl = $this->getExpiration($expire);
$key = $this->getKey($cid);
// If the item is already expired, delete it.
if ($ttl <= 0) {
$this->delete($key);
}
// Build the cache item and save it as a hash array.
$entry = $this->createEntryHash($cid, $data, $expire, $tags);
$pipe = $this->client->pipeline();
$pipe->hmset($key, $entry);
$pipe->expire($key, $ttl);
$pipe->execute();
}
/**
* {@inheritdoc}
*/
public function deleteMultiple(array $cids) {
if (!empty($cids)) {
$keys = array_map([$this, 'getKey'], $cids);
$this->client->del($keys);
}
}
}
@@ -0,0 +1,150 @@
<?php
namespace Drupal\redis\Cache;
use Drupal\Core\Cache\CacheTagsChecksumInterface;
use Drupal\Core\Cache\CacheTagsInvalidatorInterface;
use Drupal\redis\ClientFactory;
use Drupal\redis\RedisPrefixTrait;
/**
* Cache tags invalidations checksum implementation that uses redis.
*/
class RedisCacheTagsChecksum implements CacheTagsChecksumInterface, CacheTagsInvalidatorInterface {
use RedisPrefixTrait;
/**
* Contains already loaded cache invalidations from the database.
*
* @var array
*/
protected $tagCache = [];
/**
* A list of tags that have already been invalidated in this request.
*
* Used to prevent the invalidation of the same cache tag multiple times.
*
* @var array
*/
protected $invalidatedTags = [];
/**
* {@inheritdoc}
*/
protected $client;
/**
* @var string
*/
protected $clientType;
/**
* Creates a PHpRedis cache backend.
*/
public function __construct(ClientFactory $factory) {
$this->client = $factory->getClient();
$this->clientType = $factory->getClientName();
}
/**
* {@inheritdoc}
*/
public function invalidateTags(array $tags) {
$keys_to_increment = [];
foreach ($tags as $tag) {
// Only invalidate tags once per request unless they are written again.
if (isset($this->invalidatedTags[$tag])) {
continue;
}
$this->invalidatedTags[$tag] = TRUE;
unset($this->tagCache[$tag]);
$keys_to_increment[] = $this->getTagKey($tag);
}
if ($keys_to_increment) {
// We want to differentiate between PhpRedis and Redis clients.
if ($this->clientType === 'PhpRedis') {
$multi = $this->client->multi(\Redis::PIPELINE);
foreach ($keys_to_increment as $key) {
$multi->incr($key);
}
$multi->exec();
}
elseif ($this->clientType === 'Predis') {
$pipe = $this->client->pipeline();
foreach ($keys_to_increment as $key) {
$pipe->incr($key);
}
$pipe->execute();
}
}
}
/**
* {@inheritdoc}
*/
public function getCurrentChecksum(array $tags) {
// Remove tags that were already invalidated during this request from the
// static caches so that another invalidation can occur later in the same
// request. Without that, written cache items would not be invalidated
// correctly.
foreach ($tags as $tag) {
unset($this->invalidatedTags[$tag]);
}
return $this->calculateChecksum($tags);
}
/**
* {@inheritdoc}
*/
public function isValid($checksum, array $tags) {
return $checksum == $this->calculateChecksum($tags);
}
/**
* {@inheritdoc}
*/
public function calculateChecksum(array $tags) {
$checksum = 0;
$fetch = array_values(array_diff($tags, array_keys($this->tagCache)));
if ($fetch) {
$keys = array_map([$this, 'getTagKey'], $fetch);
foreach ($this->client->mget($keys) as $index => $invalidations) {
$this->tagCache[$fetch[$index]] = $invalidations ?: 0;
}
}
foreach ($tags as $tag) {
$checksum += $this->tagCache[$tag];
}
return $checksum;
}
/**
* {@inheritdoc}
*/
public function reset() {
$this->tagCache = [];
$this->invalidatedTags = [];
}
/**
* Return the key for the given cache tag.
*
* @param string $tag
* The cache tag.
*
* @return string
* The prefixed cache tag.
*/
protected function getTagKey($tag) {
return $this->getPrefix() . ':cachetags:' . $tag;
}
}
@@ -0,0 +1,97 @@
<?php
namespace Drupal\redis\Client;
use Drupal\Core\Logger\RfcLogLevel;
use Drupal\Core\Site\Settings;
use Drupal\redis\ClientInterface;
/**
* PhpRedis client specific implementation.
*/
class PhpRedis implements ClientInterface {
/**
* {@inheritdoc}
*/
public function getClient($host = NULL, $port = NULL, $base = NULL, $password = NULL) {
$client = new \Redis();
// Sentinel mode, get the real master.
if (is_array($host)) {
$ip_host = $this->askForMaster($client, $host, $password);
if (is_array($ip_host)) {
list($host, $port) = $ip_host;
}
}
$client->connect($host, $port);
if (isset($password)) {
$client->auth($password);
}
if (isset($base)) {
$client->select($base);
}
// Do not allow PhpRedis serialize itself data, we are going to do it
// ourself. This will ensure less memory footprint on Redis size when
// we will attempt to store small values.
$client->setOption(\Redis::OPT_SERIALIZER, \Redis::SERIALIZER_NONE);
return $client;
}
/**
* {@inheritdoc}
*/
public function getName() {
return 'PhpRedis';
}
/**
* Connect to sentinels to get Redis master instance.
*
* Just asking one sentinels after another until given the master location.
* More info about this mode at https://redis.io/topics/sentinel.
*
* @param \Redis $client
* The PhpRedis client.
* @param array $sentinels
* An array of the sentinels' ip:port.
* @param string $password
* An optional Sentinels' password.
*
* @return mixed
* An array with ip & port of the Master instance or NULL.
*/
protected function askForMaster(\Redis $client, array $sentinels = [], $password = NULL) {
$ip_port = NULL;
$settings = Settings::get('redis.connection', []);
$settings += ['instance' => NULL];
if ($settings['instance']) {
foreach ($sentinels as $sentinel) {
list($host, $port) = explode(':', $sentinel);
// 0.5s timeout.
$client->connect($host, $port, 0.5);
if (isset($password)) {
$client->auth($password);
}
if ($client->isConnected()) {
$ip_port = $client->rawcommand('SENTINEL', 'get-master-addr-by-name', $settings['instance']);
if ($ip_port) {
break;
}
}
$client->close();
}
}
return $ip_port;
}
}
@@ -0,0 +1,61 @@
<?php
namespace Drupal\redis\Client;
use Drupal\redis\ClientInterface;
use Predis\Client;
/**
* Predis client specific implementation.
*/
class Predis implements ClientInterface {
public function getClient($host = NULL, $port = NULL, $base = NULL, $password = NULL, $replicationHosts = NULL) {
$connectionInfo = [
'password' => $password,
'host' => $host,
'port' => $port,
'database' => $base
];
foreach ($connectionInfo as $key => $value) {
if (!isset($value)) {
unset($connectionInfo[$key]);
}
}
// I'm not sure why but the error handler is driven crazy if timezone
// is not set at this point.
// Hopefully Drupal will restore the right one this once the current
// account has logged in.
date_default_timezone_set(@date_default_timezone_get());
// If we are passed in an array of $replicationHosts, we should attempt a clustered client connection.
if ($replicationHosts !== NULL) {
$parameters = [];
foreach ($replicationHosts as $replicationHost) {
// Configure master.
if ($replicationHost['role'] === 'primary') {
$parameters[] = 'tcp://' . $replicationHost['host'] . ':' . $replicationHost['port'] . '?alias=master';
}
else {
$parameters[] = 'tcp://' . $replicationHost['host'] . ':' . $replicationHost['port'];
}
}
$options = ['replication' => true];
$client = new Client($parameters, $options);
}
else {
$client = new Client($connectionInfo);
}
return $client;
}
public function getName() {
return 'Predis';
}
}
@@ -0,0 +1,215 @@
<?php
namespace Drupal\redis;
use Drupal\Core\Site\Settings;
/**
* Common code and client singleton, for all Redis clients.
*/
class ClientFactory {
/**
* Redis default host.
*/
const REDIS_DEFAULT_HOST = "127.0.0.1";
/**
* Redis default port.
*/
const REDIS_DEFAULT_PORT = 6379;
/**
* Redis default database: will select none (Database 0).
*/
const REDIS_DEFAULT_BASE = NULL;
/**
* Redis default password: will not authenticate.
*/
const REDIS_DEFAULT_PASSWORD = NULL;
/**
* Cache implementation namespace.
*/
const REDIS_IMPL_CACHE = '\\Drupal\\redis\\Cache\\';
/**
* Lock implementation namespace.
*/
const REDIS_IMPL_LOCK = '\\Drupal\\redis\\Lock\\';
/**
* Lock implementation namespace.
*/
const REDIS_IMPL_FLOOD = '\\Drupal\\redis\\Flood\\';
/**
* Persistent Lock implementation namespace.
*/
const REDIS_IMPL_PERSISTENT_LOCK = '\\Drupal\\redis\\PersistentLock\\';
/**
* Client implementation namespace.
*/
const REDIS_IMPL_CLIENT = '\\Drupal\\redis\\Client\\';
/**
* Queue implementation namespace.
*/
const REDIS_IMPL_QUEUE = '\\Drupal\\redis\\Queue\\';
/**
* Reliable queue implementation namespace.
*/
const REDIS_IMPL_RELIABLE_QUEUE = '\\Drupal\\redis\\Queue\\Reliable';
/**
* @var \Drupal\redis\ClientInterface
*/
protected static $_clientInterface;
/**
* @var mixed
*/
protected static $_client;
public static function hasClient() {
return isset(self::$_client);
}
/**
* Set client proxy.
*/
public static function setClient(ClientInterface $interface) {
if (isset(self::$_client)) {
throw new \Exception("Once Redis client is connected, you cannot change client proxy instance.");
}
self::$_clientInterface = $interface;
}
/**
* Lazy instanciate client proxy depending on the actual configuration.
*
* If you are using a lock or cache backend using one of the Redis client
* implementations, this will be overridden at early bootstrap phase and
* configuration will be ignored.
*
* @return ClientInterface
*/
public static function getClientInterface()
{
if (!isset(self::$_clientInterface))
{
$settings = Settings::get('redis.connection', []);
if (!empty($settings['interface']))
{
$className = self::getClass(self::REDIS_IMPL_CLIENT, $settings['interface']);
self::$_clientInterface = new $className();
}
elseif (class_exists('Predis\Client'))
{
// Transparent and abitrary preference for Predis library.
$className = self::getClass(self::REDIS_IMPL_CLIENT, 'Predis');
self::$_clientInterface = new $className();
}
elseif (class_exists('Redis'))
{
// Fallback on PhpRedis if available.
$className = self::getClass(self::REDIS_IMPL_CLIENT, 'PhpRedis');
self::$_clientInterface = new $className();
}
else
{
if (!isset(self::$_clientInterface))
{
throw new \Exception("No client interface set.");
}
}
}
return self::$_clientInterface;
}
/**
* Get underlaying library name.
*
* @return string
*/
public static function getClientName() {
return self::getClientInterface()->getName();
}
/**
* Get client singleton.
*/
public static function getClient() {
if (!isset(self::$_client)) {
$settings = Settings::get('redis.connection', []);
$settings += [
'host' => self::REDIS_DEFAULT_HOST,
'port' => self::REDIS_DEFAULT_PORT,
'base' => self::REDIS_DEFAULT_BASE,
'password' => self::REDIS_DEFAULT_PASSWORD,
];
// If using replication, lets create the client appropriately.
if (isset($settings['replication']) && $settings['replication'] === TRUE) {
foreach ($settings['replication.host'] as $key => $replicationHost) {
if (!isset($replicationHost['port'])) {
$settings['replication.host'][$key]['port'] = self::REDIS_DEFAULT_PORT;
}
}
self::$_client = self::getClientInterface()->getClient(
$settings['host'],
$settings['port'],
$settings['base'],
$settings['password'],
$settings['replication.host']);
}
else {
self::$_client = self::getClientInterface()->getClient(
$settings['host'],
$settings['port'],
$settings['base'],
$settings['password']);
}
}
return self::$_client;
}
/**
* Get specific class implementing the current client usage for the specific
* asked core subsystem.
*
* @param string $system
* One of the ClientFactory::IMPL_* constant.
* @param string $clientName
* Client name, if fixed.
*
* @return string
* Class name, if found.
*
* @throws \Exception
* If not found.
*/
public static function getClass($system, $clientName = NULL) {
$className = $system . ($clientName ?: self::getClientName());
if (!class_exists($className)) {
throw new \Exception($className . " does not exists");
}
return $className;
}
/**
* For unit testing only reset internals.
*/
static public function reset() {
self::$_clientInterface = null;
self::$_client = null;
}
}
@@ -0,0 +1,26 @@
<?php
namespace Drupal\redis;
/**
* Client proxy, client handling class tied to the bare mininum.
*/
interface ClientInterface {
/**
* Get the connected client instance.
*
* @return mixed
* Real client depends from the library behind.
*/
public function getClient($host = NULL, $port = NULL, $base = NULL);
/**
* Get underlaying library name used.
*
* This can be useful for contribution code that may work with only some of
* the provided clients.
*
* @return string
*/
public function getName();
}
@@ -0,0 +1,49 @@
<?php
namespace Drupal\redis\Flood;
use Drupal\redis\ClientFactory;
use Symfony\Component\HttpFoundation\RequestStack;
/**
* Flood backend singleton handling.
*/
class FloodFactory {
/**
* @var \Drupal\redis\ClientInterface
*/
protected $clientFactory;
/**
* The request stack.
*
* @var \Symfony\Component\HttpFoundation\RequestStack
*/
protected $requestStack;
/**
* Construct the PhpRedis flood backend factory.
*
* @param \Drupal\redis\ClientFactory $client_factory
* The database connection which will be used to store the flood event
* information.
* @param \Symfony\Component\HttpFoundation\RequestStack $request_stack
* The request stack used to retrieve the current request.
*/
public function __construct(ClientFactory $client_factory, RequestStack $request_stack) {
$this->clientFactory = $client_factory;
$this->requestStack = $request_stack;
}
/**
* Get actual flood backend.
*
* @return \Drupal\Core\Flood\FloodInterface
* Return flood instance.
*/
public function get() {
$class_name = $this->clientFactory->getClass(ClientFactory::REDIS_IMPL_FLOOD);
return new $class_name($this->clientFactory, $this->requestStack);
}
}
@@ -0,0 +1,95 @@
<?php
namespace Drupal\redis\Flood;
use Drupal\Core\Flood\FloodInterface;
use Drupal\redis\ClientFactory;
use Drupal\redis\RedisPrefixTrait;
use Symfony\Component\HttpFoundation\RequestStack;
/**
* Defines the database flood backend. This is the default Drupal backend.
*/
class PhpRedis implements FloodInterface {
use RedisPrefixTrait;
/**
* @var \Redis
*/
protected $client;
/**
* The request stack.
*
* @var \Symfony\Component\HttpFoundation\RequestStack
*/
protected $requestStack;
/**
* Construct the PhpRedis flood backend.
*
* @param \Drupal\redis\ClientFactory $client_factory
* The database connection which will be used to store the flood event
* information.
* @param \Symfony\Component\HttpFoundation\RequestStack $request_stack
* The request stack used to retrieve the current request.
*/
public function __construct(ClientFactory $client_factory, RequestStack $request_stack) {
$this->client = $client_factory->getClient();
$this->requestStack = $request_stack;
}
/**
* {@inheritdoc}
*/
public function register($name, $window = 3600, $identifier = NULL) {
if (!isset($identifier)) {
$identifier = $this->requestStack->getCurrentRequest()->getClientIp();
}
$key = $this->getPrefix() . ':flood:' . $name . ':' . $identifier;
// Add a key for the event to the sorted set, the score is timestamp, so we
// can count them easily.
$this->client->zAdd($key, $_SERVER['REQUEST_TIME'] + $window, microtime(TRUE));
// Set or update the expiration for the sorted set, it will be removed if
// the newest entry expired.
$this->client->expire($key, $_SERVER['REQUEST_TIME'] + $window);
}
/**
* {@inheritdoc}
*/
public function clear($name, $identifier = NULL) {
if (!isset($identifier)) {
$identifier = $this->requestStack->getCurrentRequest()->getClientIp();
}
$key = $this->getPrefix() . ':flood:' . $name . ':' . $identifier;
$this->client->del($key);
}
/**
* {@inheritdoc}
*/
public function isAllowed($name, $threshold, $window = 3600, $identifier = NULL) {
if (!isset($identifier)) {
$identifier = $this->requestStack->getCurrentRequest()->getClientIp();
}
$key = $this->getPrefix() . ':flood:' . $name . ':' . $identifier;
// Count the in the last $window seconds.
$number = $this->client->zCount($key, $_SERVER['REQUEST_TIME'], 'inf');
return ($number < $threshold);
}
/**
* {@inheritdoc}
*/
public function garbageCollection() {
// No garbage collection necessary.
}
}
@@ -0,0 +1,95 @@
<?php
namespace Drupal\redis\Flood;
use Drupal\Core\Flood\FloodInterface;
use Drupal\redis\ClientFactory;
use Drupal\redis\RedisPrefixTrait;
use Symfony\Component\HttpFoundation\RequestStack;
/**
* Defines the database flood backend. This is the default Drupal backend.
*/
class Predis implements FloodInterface {
use RedisPrefixTrait;
/**
* @var \Predis\Client
*/
protected $client;
/**
* The request stack.
*
* @var \Symfony\Component\HttpFoundation\RequestStack
*/
protected $requestStack;
/**
* Construct the PhpRedis flood backend.
*
* @param \Drupal\redis\ClientFactory $client_factory
* The database connection which will be used to store the flood event
* information.
* @param \Symfony\Component\HttpFoundation\RequestStack $request_stack
* The request stack used to retrieve the current request.
*/
public function __construct(ClientFactory $client_factory, RequestStack $request_stack) {
$this->client = $client_factory->getClient();
$this->requestStack = $request_stack;
}
/**
* {@inheritdoc}
*/
public function register($name, $window = 3600, $identifier = NULL) {
if (!isset($identifier)) {
$identifier = $this->requestStack->getCurrentRequest()->getClientIp();
}
$key = $this->getPrefix() . ':flood:' . $name . ':' . $identifier;
// Add a key for the event to the sorted set, the score is timestamp, so we
// can count them easily.
$this->client->zAdd($key, $_SERVER['REQUEST_TIME'] + $window, microtime(TRUE));
// Set or update the expiration for the sorted set, it will be removed if
// the newest entry expired.
$this->client->expire($key, $_SERVER['REQUEST_TIME'] + $window);
}
/**
* {@inheritdoc}
*/
public function clear($name, $identifier = NULL) {
if (!isset($identifier)) {
$identifier = $this->requestStack->getCurrentRequest()->getClientIp();
}
$key = $this->getPrefix() . ':flood:' . $name . ':' . $identifier;
$this->client->del($key);
}
/**
* {@inheritdoc}
*/
public function isAllowed($name, $threshold, $window = 3600, $identifier = NULL) {
if (!isset($identifier)) {
$identifier = $this->requestStack->getCurrentRequest()->getClientIp();
}
$key = $this->getPrefix() . ':flood:' . $name . ':' . $identifier;
// Count the in the last $window seconds.
$number = $this->client->zCount($key, $_SERVER['REQUEST_TIME'], 'inf');
return ($number < $threshold);
}
/**
* {@inheritdoc}
*/
public function garbageCollection() {
// No garbage collection necessary.
}
}
@@ -0,0 +1,37 @@
<?php
namespace Drupal\redis\Lock;
use Drupal\redis\ClientFactory;
/**
* Lock backend singleton handling.
*/
class LockFactory {
/**
* @var \Drupal\redis\ClientInterface
*/
protected $clientFactory;
/**
* Creates a redis LockFactory.
*/
public function __construct(ClientFactory $client_factory) {
$this->clientFactory = $client_factory;
}
/**
* Get actual lock backend.
*
* @param bool $persistent
* (optional) Whether to return a persistent lock implementation or not.
*
* @return \Drupal\Core\Lock\LockBackendInterface
* Return lock backend instance.
*/
public function get($persistent = FALSE) {
$class_name = $this->clientFactory->getClass($persistent ? ClientFactory::REDIS_IMPL_PERSISTENT_LOCK : ClientFactory::REDIS_IMPL_LOCK);
return new $class_name($this->clientFactory);
}
}
@@ -0,0 +1,147 @@
<?php
namespace Drupal\redis\Lock;
use Drupal\Core\Lock\LockBackendAbstract;
use Drupal\redis\ClientFactory;
use Drupal\redis\RedisPrefixTrait;
/**
* Predis lock backend implementation.
*/
class PhpRedis extends LockBackendAbstract {
use RedisPrefixTrait;
/**
* @var \Redis
*/
protected $client;
/**
* Creates a PHpRedis cache backend.
*/
public function __construct(ClientFactory $factory) {
$this->client = $factory->getClient();
// __destruct() is causing problems with garbage collections, register a
// shutdown function instead.
drupal_register_shutdown_function([$this, 'releaseAll']);
}
/**
* Generate a redis key name for the current lock name.
*
* @param string $name
* Lock name.
*
* @return string
* The redis key for the given lock.
*/
protected function getKey($name) {
return $this->getPrefix() . ':lock:' . $name;
}
/**
* {@inheritdoc}
*/
public function acquire($name, $timeout = 30.0) {
$key = $this->getKey($name);
$id = $this->getLockId();
// Insure that the timeout is at least 1 ms.
$timeout = max($timeout, 0.001);
// If we already have the lock, check for his owner and attempt a new EXPIRE
// command on it.
if (isset($this->locks[$name])) {
// Create a new transaction, for atomicity.
$this->client->watch($key);
// Global tells us we are the owner, but in real life it could have expired
// and another process could have taken it, check that.
if ($this->client->get($key) != $id) {
// Explicit UNWATCH we are not going to run the MULTI/EXEC block.
$this->client->unwatch();
unset($this->locks[$name]);
return FALSE;
}
$result = $this->client->multi()
->psetex($key, (int) ($timeout * 1000), $id)
->exec();
// If the set failed, someone else wrote the key, we failed to acquire
// the lock.
if (FALSE === $result) {
unset($this->locks[$name]);
// Explicit transaction release which also frees the WATCH'ed key.
$this->client->discard();
return FALSE;
}
return ($this->locks[$name] = TRUE);
}
else {
// Use a SET with microsecond expiration and the NX flag, which will only
// succeed if the key does not exist yet.
$result = $this->client->set($key, $id, ['nx', 'px' => (int) ($timeout * 1000)]);
// If the result is FALSE, we failed to acquire the lock.
if (FALSE === $result) {
return FALSE;
}
// Register the lock.
return ($this->locks[$name] = TRUE);
}
}
/**
* {@inheritdoc}
*/
public function lockMayBeAvailable($name) {
$key = $this->getKey($name);
$value = $this->client->get($key);
// In Drupal 7, this method treated the lock as available if the ID did
// match. The database backend and test expects it to return FALSE in that
// case, updated accordingly.
return FALSE === $value;
}
/**
* {@inheritdoc}
*/
public function release($name) {
$key = $this->getKey($name);
$id = $this->getLockId();
unset($this->locks[$name]);
// Ensure the lock deletion is an atomic transaction. If another thread
// manages to removes all lock, we can not alter it anymore else we will
// release the lock for the other thread and cause race conditions.
$this->client->watch($key);
if ($this->client->get($key) == $id) {
$this->client->multi();
$this->client->delete($key);
$this->client->exec();
}
else {
$this->client->unwatch();
}
}
/**
* {@inheritdoc}
*/
public function releaseAll($lock_id = NULL) {
// We can afford to deal with a slow algorithm here, this should not happen
// on normal run because we should have removed manually all our locks.
foreach ($this->locks as $name => $foo) {
$this->release($name);
}
}
}
@@ -0,0 +1,136 @@
<?php
namespace Drupal\redis\Lock;
use Drupal\Core\Lock\LockBackendAbstract;
use Drupal\redis\ClientFactory;
use Drupal\redis\RedisPrefixTrait;
/**
* Predis lock backend implementation.
*/
class Predis extends LockBackendAbstract {
use RedisPrefixTrait;
/**
* @var \Predis\Client
*/
protected $client;
/**
* Creates a PHpRedis cache backend.
*/
public function __construct(ClientFactory $factory) {
$this->client = $factory->getClient();
// __destruct() is causing problems with garbage collections, register a
// shutdown function instead.
drupal_register_shutdown_function([$this, 'releaseAll']);
}
/**
* Generate a redis key name for the current lock name.
*
* @param string $name
* Lock name.
*
* @return string
* The redis key for the given lock.
*/
protected function getKey($name) {
return $this->getPrefix() . ':lock:' . $name;
}
public function acquire($name, $timeout = 30.0) {
$key = $this->getKey($name);
$id = $this->getLockId();
// Insure that the timeout is at least 1 ms.
$timeout = max($timeout, 0.001);
// If we already have the lock, check for his owner and attempt a new EXPIRE
// command on it.
if (isset($this->locks[$name])) {
// Create a new transaction, for atomicity.
$this->client->watch($key);
// Global tells us we are the owner, but in real life it could have expired
// and another process could have taken it, check that.
if ($this->client->get($key) != $id) {
// Explicit UNWATCH we are not going to run the MULTI/EXEC block.
$this->client->unwatch();
unset($this->locks[$name]);
return FALSE;
}
$result = $this->client->pipeline()
->psetex($key, (int) ($timeout * 1000), $id)
->exec();
// If the set failed, someone else wrote the key, we failed to acquire
// the lock.
if (FALSE === $result) {
unset($this->locks[$name]);
// Explicit transaction release which also frees the WATCH'ed key.
$this->client->discard();
return FALSE;
}
return ($this->locks[$name] = TRUE);
}
else {
// Use a SET with microsecond expiration and the NX flag, which will only
// succeed if the key does not exist yet.
$result = $this->client->set($key, $id, 'nx', 'px', (int) ($timeout * 1000));
// If the result is FALSE, we failed to acquire the lock.
if (FALSE === $result) {
return FALSE;
}
// Register the lock.
return ($this->locks[$name] = TRUE);
}
}
public function lockMayBeAvailable($name) {
$key = $this->getKey($name);
$value = $this->client->get($key);
// In Drupal 7, this method treated the lock as available if the ID did
// match. The database backend and test expects it to return FALSE in that
// case, updated accordingly.
return FALSE === $value;
}
public function release($name) {
$key = $this->getKey($name);
$id = $this->getLockId();
unset($this->locks[$name]);
// Ensure the lock deletion is an atomic transaction. If another thread
// manages to removes all lock, we can not alter it anymore else we will
// release the lock for the other thread and cause race conditions.
$this->client->watch($key);
if ($this->client->get($key) == $id) {
$pipe = $this->client->pipeline();
$pipe->del([$key]);
$pipe->execute();
}
else {
$this->client->unwatch();
}
}
public function releaseAll($lock_id = NULL) {
// We can afford to deal with a slow algorithm here, this should not happen
// on normal run because we should have removed manually all our locks.
foreach ($this->locks as $name => $foo) {
$this->release($name);
}
}
}
@@ -0,0 +1,26 @@
<?php
namespace Drupal\redis\PersistentLock;
use Drupal\redis\ClientFactory;
/**
* PHpRedis persistent lock backend
*/
class PhpRedis extends \Drupal\redis\Lock\PhpRedis {
/**
* Creates a PHpRedis persistent lock backend.
*/
public function __construct(ClientFactory $factory) {
// Do not call the parent constructor to avoid registering a shutdown
// function that releases all the locks at the end of a request.
$this->client = $factory->getClient();
// Set the lockId to a fixed string to make the lock ID the same across
// multiple requests. The lock ID is used as a page token to relate all the
// locks set during a request to each other.
// @see \Drupal\Core\Lock\LockBackendInterface::getLockId()
$this->lockId = 'persistent';
}
}
@@ -0,0 +1,148 @@
<?php
namespace Drupal\redis\Queue;
/**
* Redis queue implementation using PhpRedis extension backend.
*
* @ingroup queue
*/
class PhpRedis extends QueueBase {
/**
* The Redis connection.
*
* @var \Redis $client
*/
protected $client;
/**
* Constructs a \Drupal\redis\Queue\PhpRedis object.
*
* @param string $name
* The name of the queue.
* @param array $settings
* Array of Redis-related settings for this queue.
* @param \Redis $client
* The PhpRedis client.
*/
public function __construct($name, array $settings, \Redis $client) {
parent::__construct($name, $settings);
$this->client = $client;
}
/**
* {@inheritdoc}
*/
public function createItem($data) {
$record = new \stdClass();
$record->data = $data;
$record->qid = $this->incrementId();
// We cannot rely on REQUEST_TIME because many items might be created
// by a single request which takes longer than 1 second.
$record->timestamp = time();
if (!$this->client->hsetnx($this->availableItems, $record->qid, serialize($record))) {
return FALSE;
}
$start_len = $this->client->lLen($this->availableListKey);
if ($start_len < $this->client->lpush($this->availableListKey, $record->qid)) {
return $record->qid;
}
return FALSE;
}
/**
* Gets next serial ID for Redis queue items.
*
* @return int
* Next serial ID for Redis queue item.
*/
protected function incrementId() {
return $this->client->incr($this->incrementCounterKey);
}
/**
* {@inheritdoc}
*/
public function numberOfItems() {
return $this->client->lLen($this->availableListKey);
}
/**
* {@inheritdoc}
*/
public function claimItem($lease_time = 30) {
// Is it OK to do garbage collection here (we need to loop list of claimed
// items)?
$this->garbageCollection();
$item = FALSE;
if ($this->reserveTimeout !== NULL) {
// A blocking version of claimItem to be used with long-running queue workers.
$qid = $this->client->brpoplpush($this->availableListKey, $this->claimedListKey, $this->reserveTimeout);
}
else {
$qid = $this->client->rpoplpush($this->availableListKey, $this->claimedListKey);
}
if ($qid) {
$job = $this->client->hget($this->availableItems, $qid);
if ($job) {
$item = unserialize($job);
$this->client->setex($this->leasedKeyPrefix . $item->qid, $lease_time, '1');
}
}
return $item;
}
/**
* {@inheritdoc}
*/
public function releaseItem($item) {
$this->client->lrem($this->claimedListKey, $item->qid, -1);
$this->client->lpush($this->availableListKey, $item->qid);
}
/**
* {@inheritdoc}
*/
public function deleteItem($item) {
$this->client->lrem($this->claimedListKey, $item->qid, -1);
$this->client->hdel($this->availableItems, $item->qid);
}
/**
* {@inheritdoc}
*/
public function deleteQueue() {
$keys_to_remove = [
$this->claimedListKey,
$this->availableListKey,
$this->availableItems,
$this->incrementCounterKey
];
foreach ($this->client->keys($this->leasedKeyPrefix . '*') as $key) {
$keys_to_remove[] = $key;
}
$this->client->del($keys_to_remove);
}
/**
* Automatically release items, that have been claimed and exceeded lease time.
*/
protected function garbageCollection() {
foreach ($this->client->lrange($this->claimedListKey, 0, -1) as $qid) {
if (!$this->client->exists($this->leasedKeyPrefix . $qid)) {
// The lease expired for this ID.
$this->client->lrem($this->claimedListKey, $qid, -1);
$this->client->lpush($this->availableListKey, $qid);
}
}
}
}
@@ -0,0 +1,152 @@
<?php
namespace Drupal\redis\Queue;
/**
* Redis queue implementation using Predis library backend.
*
* @ingroup queue
*/
class Predis extends QueueBase {
/**
* The Redis connection.
*
* @var \Predis\Client $client
*/
protected $client;
/**
* Constructs a \Drupal\redis\Queue\Predis object.
*
* @param string $name
* The name of the queue.
* @param array $settings
* Array of Redis-related settings for this queue.
* @param \Predis\Client $client
* The Predis client.
*/
public function __construct($name, array $settings, \Predis\Client $client) {
parent::__construct($name, $settings);
$this->client = $client;
}
/**
* {@inheritdoc}
*/
public function createItem($data) {
// TODO: Fixme
$record = new \stdClass();
$record->data = $data;
$record->qid = $this->incrementId();
// We cannot rely on REQUEST_TIME because many items might be created
// by a single request which takes longer than 1 second.
$record->timestamp = time();
if (!$this->client->hsetnx($this->availableItems, $record->qid, serialize($record))) {
return FALSE;
}
$start_len = $this->client->lLen($this->availableListKey);
if ($start_len < $this->client->lpush($this->availableListKey, $record->qid)) {
return $record->qid;
}
}
/**
* Gets next serial ID for Redis queue items.
*
* @return int
* Next serial ID for Redis queue item.
*/
protected function incrementId() {
// TODO: Fixme
return $this->client->incr($this->incrementCounterKey);
}
/**
* {@inheritdoc}
*/
public function numberOfItems() {
// TODO: Fixme
return $this->client->lLen($this->availableListKey);
}
/**
* {@inheritdoc}
*/
public function claimItem($lease_time = 30) {
// Is it OK to do garbage collection here (we need to loop list of claimed
// items)?
$this->garbageCollection();
$item = FALSE;
if ($this->reserveTimeout !== NULL) {
// A blocking version of claimItem to be used with long-running queue workers.
$qid = $this->client->brpoplpush($this->availableListKey, $this->claimedListKey, $this->reserveTimeout);
}
else {
$qid = $this->client->rpoplpush($this->availableListKey, $this->claimedListKey);
}
if ($qid) {
$job = $this->client->hget($this->availableItems, $qid);
if ($job) {
$item = unserialize($job);
$this->client->setex($this->leasedKeyPrefix . $item->qid, $lease_time, '1');
}
}
return $item;
}
/**
* {@inheritdoc}
*/
public function releaseItem($item) {
// TODO: Fixme
$this->client->lrem($this->claimedListKey, $item->qid, -1);
$this->client->lpush($this->availableListKey, $item->qid);
}
/**
* {@inheritdoc}
*/
public function deleteItem($item) {
// TODO: Fixme
$this->client->lrem($this->claimedListKey, $item->qid, -1);
$this->client->hdel($this->availableItems, $item->qid);
}
/**
* {@inheritdoc}
*/
public function deleteQueue() {
// TODO: Fixme
$keys_to_remove = [
$this->claimedListKey,
$this->availableListKey,
$this->availableItems,
$this->incrementCounterKey
];
foreach ($this->client->keys($this->leasedKeyPrefix . '*') as $key) {
$keys_to_remove[] = $key;
}
$this->client->del($keys_to_remove);
}
/**
* Automatically release items, that have been claimed and exceeded lease time.
*/
protected function garbageCollection() {
foreach ($this->client->lrange($this->claimedListKey, 0, -1) as $qid) {
if (!$this->client->exists($this->leasedKeyPrefix . $qid)) {
// The lease expired for this ID.
$this->client->lrem($this->claimedListKey, $qid, -1);
$this->client->lpush($this->availableListKey, $qid);
}
}
}
}
@@ -0,0 +1,96 @@
<?php
namespace Drupal\redis\Queue;
use Drupal\Core\Queue\QueueInterface;
/**
* Redis queue implementation.
*
* @ingroup queue
*/
abstract class QueueBase implements QueueInterface {
/**
* Prefix used with all keys.
*/
const KEY_PREFIX = 'drupal:queue:';
/**
* The name of the queue this instance is working with.
*
* @var string
*/
protected $name;
/**
* Key for list of available items.
*
* @var string
*/
protected $availableListKey;
/**
* Key for list of claimed items.
*
* @var string
*/
protected $claimedListKey;
/**
* Key prefix for items that are used to track expiration of leased items.
*
* @var string
*/
protected $leasedKeyPrefix;
/**
* Key of increment counter key.
*
* @var string
*/
protected $incrementCounterKey;
/**
* Key for hash table of available queue items.
*
* @var string
*/
protected $availableItems;
/**
* Reserve timeout for blocking item claim.
*
* This will be set to number of seconds to wait for an item to be claimed.
* Non-blocking approach will be used when set to NULL.
*
* @var int|null
*/
protected $reserveTimeout;
/**
* Constructs a \Drupal\Core\Queue\DatabaseQueue object.
*
* @param string $name
* The name of the queue.
* @param array $settings
* Array of Redis-related settings for this queue.
*/
public function __construct($name, array $settings) {
$this->name = $name;
$this->reserveTimeout = $settings['reserve_timeout'];
$this->availableListKey = static::KEY_PREFIX . $name . ':avail';
$this->availableItems = static::KEY_PREFIX . $name . ':items';
$this->claimedListKey = static::KEY_PREFIX . $name . ':claimed';
$this->leasedKeyPrefix = static::KEY_PREFIX . $name . ':lease:';
$this->incrementCounterKey = static::KEY_PREFIX . $name . ':counter';
}
/**
* {@inheritdoc}
*/
public function createQueue() {
// Nothing to do here.
}
}
@@ -0,0 +1,56 @@
<?php
namespace Drupal\redis\Queue;
use Drupal\Core\Site\Settings;
use Drupal\redis\ClientFactory;
/**
* Defines the queue factory for the Redis backend.
*/
class QueueRedisFactory {
/**
* Queue implementation class namespace prefix.
*/
const CLASS_NAMESPACE = ClientFactory::REDIS_IMPL_QUEUE;
/**
* @var \Drupal\redis\ClientFactory
*/
protected $clientFactory;
/**
* The settings array.
*
* @var \Drupal\Core\Site\Settings
*/
protected $settings;
/**
* Constructs this factory object.
*
* @param \Drupal\Core\Database\Connection $connection
* The Connection object containing the key-value tables.
*/
public function __construct(ClientFactory $client_factory, Settings $settings) {
$this->clientFactory = $client_factory;
$this->settings = $settings;
}
/**
* Constructs a new queue object for a given name.
*
* @param string $name
* The name of the collection holding key and value pairs.
*
* @return \Drupal\Core\Queue\DatabaseQueue
* A key/value store implementation for the given $collection.
*/
public function get($name) {
$settings = $this->settings->get('redis_queue_' . $name, ['reserve_timeout' => NULL]);
$class_name = $this->clientFactory->getClass(static::CLASS_NAMESPACE);
return new $class_name($name, $settings, $this->clientFactory->getClient());
}
}
@@ -0,0 +1,153 @@
<?php
namespace Drupal\redis\Queue;
/**
* Redis queue implementation using PhpRedis extension backend.
*
* @ingroup queue
*/
class ReliablePhpRedis extends ReliableQueueBase {
/**
* The Redis connection.
*
* @var \Redis $client
*/
protected $client;
/**
* Constructs a \Drupal\redis\Queue\PhpRedis object.
*
* @param string $name
* The name of the queue.
* @param array $settings
* Array of Redis-related settings for this queue.
* @param \Redis $client
* The PhpRedis client.
*/
public function __construct($name, array $settings, \Redis $client) {
parent::__construct($name, $settings);
$this->client = $client;
}
/**
* {@inheritdoc}
*/
public function createItem($data) {
$record = new \stdClass();
$record->data = $data;
$record->qid = $this->incrementId();
// We cannot rely on REQUEST_TIME because many items might be created
// by a single request which takes longer than 1 second.
$record->timestamp = time();
$result = $this->client->multi()
->hsetnx($this->availableItems, $record->qid, serialize($record))
->lLen($this->availableListKey)
->lpush($this->availableListKey, $record->qid)
->exec();
$success = $result[0] && $result[2] > $result[1];
return $success ? $record->qid : FALSE;
}
/**
* Gets next serial ID for Redis queue items.
*
* @return int
* Next serial ID for Redis queue item.
*/
protected function incrementId() {
return $this->client->incr($this->incrementCounterKey);
}
/**
* {@inheritdoc}
*/
public function numberOfItems() {
return $this->client->lLen($this->availableListKey);
}
/**
* {@inheritdoc}
*/
public function claimItem($lease_time = 30) {
// Is it OK to do garbage collection here (we need to loop list of claimed
// items)?
$this->garbageCollection();
$item = FALSE;
if ($this->reserveTimeout !== NULL) {
// A blocking version of claimItem to be used with long-running queue workers.
$qid = $this->client->brpoplpush($this->availableListKey, $this->claimedListKey, $this->reserveTimeout);
}
else {
$qid = $this->client->rpoplpush($this->availableListKey, $this->claimedListKey);
}
if ($qid) {
$job = $this->client->hget($this->availableItems, $qid);
if ($job) {
$item = unserialize($job);
$this->client->setex($this->leasedKeyPrefix . $item->qid, $lease_time, '1');
}
}
return $item;
}
/**
* {@inheritdoc}
*/
public function releaseItem($item) {
$this->client->multi()
->lrem($this->claimedListKey, $item->qid, -1)
->lpush($this->availableListKey, $item->qid)
->exec();
}
/**
* {@inheritdoc}
*/
public function deleteItem($item) {
$this->client->multi()
->lrem($this->claimedListKey, $item->qid, -1)
->hdel($this->availableItems, $item->qid)
->exec();
}
/**
* {@inheritdoc}
*/
public function deleteQueue() {
$keys_to_remove = [
$this->claimedListKey,
$this->availableListKey,
$this->availableItems,
$this->incrementCounterKey
];
foreach ($this->client->keys($this->leasedKeyPrefix . '*') as $key) {
$keys_to_remove[] = $key;
}
$this->client->del($keys_to_remove);
}
/**
* Automatically release items, that have been claimed and exceeded lease time.
*/
protected function garbageCollection() {
foreach ($this->client->lrange($this->claimedListKey, 0, -1) as $qid) {
if (!$this->client->exists($this->leasedKeyPrefix . $qid)) {
// The lease expired for this ID.
$this->client->multi()
->lrem($this->claimedListKey, $qid, -1)
->lpush($this->availableListKey, $qid)
->exec();
}
}
}
}
@@ -0,0 +1,156 @@
<?php
namespace Drupal\redis\Queue;
/**
* Redis queue implementation using Predis library backend.
*
* @ingroup queue
*/
class ReliablePredis extends ReliableQueueBase {
/**
* The Redis connection.
*
* @var \Predis\Client $client
*/
protected $client;
/**
* Constructs a \Drupal\redis\Queue\Predis object.
*
* @param string $name
* The name of the queue.
* @param array $settings
* Array of Redis-related settings for this queue.
* @param \Predis\Client $client
* The Predis client.
*/
public function __construct($name, array $settings, \Predis\Client $client) {
parent::__construct($name, $settings);
$this->client = $client;
}
/**
* {@inheritdoc}
*/
public function createItem($data) {
$record = new \stdClass();
$record->data = $data;
$record->qid = $this->incrementId();
// We cannot rely on REQUEST_TIME because many items might be created
// by a single request which takes longer than 1 second.
$record->timestamp = time();
$pipe = $this->client->pipeline();
$pipe->hsetnx($this->availableItems, $record->qid, serialize($record));
$pipe->lLen($this->availableListKey);
$pipe->lpush($this->availableListKey, $record->qid);
$result = $pipe->execute();
$success = $result[0] && $result[2] > $result[1];
return $success ? $record->qid : FALSE;
}
/**
* Gets next serial ID for Redis queue items.
*
* @return int
* Next serial ID for Redis queue item.
*/
protected function incrementId() {
// TODO: Fixme
return $this->client->incr($this->incrementCounterKey);
}
/**
* {@inheritdoc}
*/
public function numberOfItems() {
// TODO: Fixme
return $this->client->lLen($this->availableListKey);
}
/**
* {@inheritdoc}
*/
public function claimItem($lease_time = 30) {
// Is it OK to do garbage collection here (we need to loop list of claimed
// items)?
$this->garbageCollection();
$item = FALSE;
if ($this->reserveTimeout !== NULL) {
// A blocking version of claimItem to be used with long-running queue workers.
$qid = $this->client->brpoplpush($this->availableListKey, $this->claimedListKey, $this->reserveTimeout);
}
else {
$qid = $this->client->rpoplpush($this->availableListKey, $this->claimedListKey);
}
if ($qid) {
$job = $this->client->hget($this->availableItems, $qid);
if ($job) {
$item = unserialize($job);
$this->client->setex($this->leasedKeyPrefix . $item->qid, $lease_time, '1');
}
}
return $item;
}
/**
* {@inheritdoc}
*/
public function releaseItem($item) {
// TODO: Fixme
$this->client->pipeline()
->lrem($this->claimedListKey, $item->qid, -1)
->lpush($this->availableListKey, $item->qid)
->exec();
}
/**
* {@inheritdoc}
*/
public function deleteItem($item) {
// TODO: Fixme
$this->client->pipeline()
->lrem($this->claimedListKey, $item->qid, -1)
->hdel($this->availableItems, $item->qid)
->exec();
}
/**
* {@inheritdoc}
*/
public function deleteQueue() {
// TODO: Fixme
$keys_to_remove = [
$this->claimedListKey,
$this->availableListKey,
$this->availableItems,
$this->incrementCounterKey
];
foreach ($this->client->keys($this->leasedKeyPrefix . '*') as $key) {
$keys_to_remove[] = $key;
}
$this->client->del($keys_to_remove);
}
/**
* Automatically release items, that have been claimed and exceeded lease time.
*/
protected function garbageCollection() {
foreach ($this->client->lrange($this->claimedListKey, 0, -1) as $qid) {
if (!$this->client->exists($this->leasedKeyPrefix . $qid)) {
// The lease expired for this ID.
$this->client->lrem($this->claimedListKey, $qid, -1);
$this->client->lpush($this->availableListKey, $qid);
}
}
}
}
@@ -0,0 +1,14 @@
<?php
namespace Drupal\redis\Queue;
use Drupal\Core\Queue\ReliableQueueInterface;
/**
* Redis queue implementation.
*
* @ingroup queue
*/
abstract class ReliableQueueBase extends QueueBase implements ReliableQueueInterface {
}
@@ -0,0 +1,17 @@
<?php
namespace Drupal\redis\Queue;
use Drupal\redis\ClientFactory;
/**
* Defines the queue factory for the Redis backend.
*/
class ReliableQueueRedisFactory extends QueueRedisFactory {
/**
* Queue implementation class namespace prefix.
*/
const CLASS_NAMESPACE = ClientFactory::REDIS_IMPL_RELIABLE_QUEUE;
}
@@ -0,0 +1,89 @@
<?php
namespace Drupal\redis;
use Drupal\Core\Site\Settings;
trait RedisPrefixTrait {
/**
* @var string
*/
protected $prefix;
/**
* Get global default prefix
*
* @param string $suffix
*
* @return string
*/
protected function getDefaultPrefix($suffix = NULL) {
$ret = NULL;
if ($test_prefix = drupal_valid_test_ua()) {
$ret = $test_prefix;
}
else {
$prefixes = Settings::get('cache_prefix', '');
if (is_string($prefixes)) {
// Variable can be a string which then considered as a default
// behavior.
$ret = $prefixes;
}
else if (NULL !== $suffix && isset($prefixes[$suffix])) {
if (FALSE !== $prefixes[$suffix]) {
// If entry is set and not false an explicit prefix is set
// for the bin.
$ret = $prefixes[$suffix];
}
else {
// If we have an explicit false it means no prefix whatever
// is the default configuration.
$ret = '';
}
}
else {
// Key is not set, we can safely rely on default behavior.
if (isset($prefixes['default']) && FALSE !== $prefixes['default']) {
$ret = $prefixes['default'];
}
else {
// When default is not set or an explicit false this means
// no prefix.
$ret = '';
}
}
}
if (empty($ret)) {
// If no prefix is given, use the same logic as core for APCu caching.
$ret = Settings::getApcuPrefix('redis', DRUPAL_ROOT);
}
return $ret;
}
/**
* Set prefix
*
* @param string $prefix
*/
public function setPrefix($prefix) {
$this->prefix = $prefix;
}
/**
* Get prefix
*
* @return string
*/
protected function getPrefix() {
if (!isset($this->prefix)) {
$this->prefix = $this->getDefaultPrefix();
}
return $this->prefix;
}
}