From 6bbf9c3dab1caee567b65b233f9533804749c99d Mon Sep 17 00:00:00 2001 From: Netkas Date: Mon, 19 Jun 2023 17:29:42 -0400 Subject: [PATCH] Updated ClientManager to implement the cache system. --- src/FederationCLI/InteractiveMode.php | 4 +- src/FederationLib/Classes/Configuration.php | 36 ++ src/FederationLib/FederationLib.php | 24 +- src/FederationLib/Managers/ClientManager.php | 350 +++++++++++------- .../Managers/RedisConnectionManager.php | 1 - 5 files changed, 256 insertions(+), 159 deletions(-) diff --git a/src/FederationCLI/InteractiveMode.php b/src/FederationCLI/InteractiveMode.php index 470e031..c9b0369 100644 --- a/src/FederationCLI/InteractiveMode.php +++ b/src/FederationCLI/InteractiveMode.php @@ -38,8 +38,8 @@ */ public static function main(array $args=[]): void { - tm::initialize(TamerMode::CLIENT, Configuration::getTamerLibConfiguration()->getServerConfiguration()); - tm::createWorker(Configuration::getTamerLibConfiguration()->getCliWorkers(), FederationLib::getSubprocessorPath()); + tm::initialize(TamerMode::CLIENT); + tm::createWorker(Configuration::getTamerLibConfiguration()->getCliWorkers(), FederationLib::getSubprocessPath()); self::$federation_lib = new FederationLib(); diff --git a/src/FederationLib/Classes/Configuration.php b/src/FederationLib/Classes/Configuration.php index 87320df..57df17c 100644 --- a/src/FederationLib/Classes/Configuration.php +++ b/src/FederationLib/Classes/Configuration.php @@ -313,4 +313,40 @@ { return (int)self::getConfiguration()['cache_system']['error_connection_priority']; } + + /** + * @param string $name + * @return bool + */ + public static function getObjectCacheEnabled(string $name): bool + { + return (bool)self::getConfiguration()['cache_system']['cache'][sprintf('%s_enabled', $name)]; + } + + /** + * @param string $name + * @return int + */ + public static function getObjectCacheTtl(string $name): int + { + return (int)self::getConfiguration()['cache_system']['cache'][sprintf('%s_ttl', $name)]; + } + + /** + * @param string $name + * @return string + */ + public static function getObjectCacheServerPreference(string $name): string + { + return self::getConfiguration()['cache_system']['cache'][sprintf('%s_server_preference', $name)]; + } + + /** + * @param string $name + * @return string + */ + public static function getObjectCacheServerFallback(string $name): string + { + return self::getConfiguration()['cache_system']['cache'][sprintf('%s_server_fallback', $name)]; + } } \ No newline at end of file diff --git a/src/FederationLib/FederationLib.php b/src/FederationLib/FederationLib.php index 3919167..ffe5eba 100755 --- a/src/FederationLib/FederationLib.php +++ b/src/FederationLib/FederationLib.php @@ -4,7 +4,6 @@ use Exception; use FederationLib\Classes\Configuration; - use FederationLib\Enums\Misc; use FederationLib\Enums\Standard\ErrorCodes; use FederationLib\Enums\Standard\Methods; use FederationLib\Exceptions\DatabaseException; @@ -17,7 +16,6 @@ use FederationLib\Objects\Client; use FederationLib\Objects\ResolvedIdentity; use FederationLib\Objects\Standard\ClientIdentity; - use LogLib\Log; use TamerLib\Enums\TamerMode; use TamerLib\tm; use Throwable; @@ -52,15 +50,23 @@ $this->client_manager->registerFunctions(); } + /** + * @return string + */ + public static function getSubprocessPath(): string + { + return __DIR__ . DIRECTORY_SEPARATOR . 'subproc'; + } + /** * Resolves the permission role from the given identity and attempts to check if the identity has the * required permission to perform the given method * * @param ClientIdentity|null $identity - * @return ResolvedIdentity * @throws AccessDeniedException * @throws ClientNotFoundException * @throws InternalServerException + * @return ResolvedIdentity */ private function resolveIdentity(?ClientIdentity $identity): ResolvedIdentity { @@ -75,11 +81,12 @@ try { $client = tm::waitFor($get_client); + tm::dof('client_updateLastSeen'); } catch(ClientNotFoundException $e) { tm::clear(); - throw new ClientNotFoundException('The client you are trying to access does not exist', $e); + throw new ClientNotFoundException('Invalid client UUID or client does not exist', $e); } catch(Exception|Throwable $e) { @@ -87,7 +94,6 @@ throw new InternalServerException('There was an error while trying to access the client', $e); } - tm::dof('client_updateLastSeen'); return new ResolvedIdentity($client, $peer); } @@ -258,12 +264,4 @@ return true; } - /** - * @return string - */ - public static function getSubprocessorPath(): string - { - return __DIR__ . DIRECTORY_SEPARATOR . 'subproc'; - } - } \ No newline at end of file diff --git a/src/FederationLib/Managers/ClientManager.php b/src/FederationLib/Managers/ClientManager.php index d1af6a1..3c58ab6 100644 --- a/src/FederationLib/Managers/ClientManager.php +++ b/src/FederationLib/Managers/ClientManager.php @@ -6,13 +6,13 @@ use Exception; use FederationLib\Classes\Configuration; use FederationLib\Classes\Database; - use FederationLib\Classes\Redis; use FederationLib\Classes\Security; use FederationLib\Classes\Utilities; use FederationLib\Classes\Validate; use FederationLib\Enums\DatabaseTables; use FederationLib\Enums\FilterOrder; use FederationLib\Enums\Filters\ListClientsFilter; + use FederationLib\Enums\Misc; use FederationLib\Exceptions\DatabaseException; use FederationLib\Exceptions\Standard\ClientNotFoundException; use FederationLib\Exceptions\Standard\InvalidClientDescriptionException; @@ -42,6 +42,11 @@ $this->federationLib = $federationLib; } + /** + * Registers functions to the TamerLib instance, if applicable + * + * @return void + */ public function registerFunctions(): void { if(tm::getMode() !== TamerMode::WORKER) @@ -54,7 +59,6 @@ tm::addFunction('client_changeClientName', [$this, 'changeClientName']); tm::addFunction('client_changeClientDescription', [$this, 'changeClientDescription']); tm::addFunction('client_changeClientPermissionRole', [$this, 'changeClientPermissionRole']); - tm::addFunction('client_updateClient', [$this, 'updateClient']); tm::addFunction('client_updateLastSeen', [$this, 'updateLastSeen']); tm::addFunction('client_listClients', [$this, 'listClients']); tm::addFunction('client_getTotalClients', [$this, 'getTotalClients']); @@ -82,12 +86,10 @@ { $name = Utilities::generateName(4); } - else + + if(!Validate::clientName($name)) { - if(!Validate::clientName($name)) - { - throw new InvalidClientNameException(sprintf('Invalid client name: %s', $name)); - } + throw new InvalidClientNameException(sprintf('Invalid client name: %s', $name)); } if($description !== null && strlen($description) > 128) @@ -129,6 +131,29 @@ $client_uuid = $client_uuid->getUuid(); } + // Use the cache first if it's enabled + if(Configuration::isCacheSystemEnabled() && Configuration::getObjectCacheEnabled('client_objects')) + { + try + { + $redis = RedisConnectionManager::getConnection( + Configuration::getObjectCacheServerPreference('client_objects'), + Configuration::getObjectCacheServerFallback('client_objects') + ); + + if($redis->exists($client_uuid)) + { + $client = Client::fromArray($redis->hGetAll($client_uuid)); + Log::debug(Misc::FEDERATIONLIB, sprintf('Loaded client object %s from cache', $client_uuid)); + return $client; + } + } + catch(Exception $e) + { + Log::warning(Misc::FEDERATIONLIB, sprintf('Failed to load client object %s from cache: %s', $client_uuid, $e->getMessage())); + } + } + $qb = Database::getConnection()->createQueryBuilder(); $qb->select('*'); $qb->from(DatabaseTables::CLIENTS); @@ -156,6 +181,30 @@ throw new DatabaseException('Failed to get Client: ' . $e->getMessage(), $e); } + // Store the record in the cache if caching is enabled. + if(Configuration::isCacheSystemEnabled() && Configuration::getObjectCacheEnabled('client_objects')) + { + try + { + $redis = RedisConnectionManager::getConnection( + Configuration::getObjectCacheServerPreference('client_objects'), + Configuration::getObjectCacheServerFallback('client_objects') + ); + + $redis->hMSet($client->getUuid(), $client->toArray()); + if(Configuration::getObjectCacheTTL('client_objects') > 0) + { + $redis->expire($client->getUuid(), Configuration::getObjectCacheTTL('client_objects')); + } + + Log::debug(Misc::FEDERATIONLIB, sprintf('Cached client object %s', $client->getUuid())); + } + catch(Exception $e) + { + Log::warning(Misc::FEDERATIONLIB, sprintf('Failed to cache client object %s: %s', $client->getUuid(), $e->getMessage()), $e); + } + } + return $client; } @@ -180,20 +229,19 @@ { $name = Utilities::generateName(4); } - else + + if(!Validate::clientName($name)) { - if(!Validate::clientName($name)) - { - throw new InvalidClientNameException(sprintf('Invalid client name: %s', $name)); - } + throw new InvalidClientNameException(sprintf('Invalid client name: %s', $name)); } + $update_timestamp = time(); $qb = Database::getConnection()->createQueryBuilder(); $qb->update(DatabaseTables::CLIENTS); $qb->set('name', ':name'); $qb->setParameter('name', $name); $qb->set('updated_timestamp', ':updated_timestamp'); - $qb->setParameter('updated_timestamp', time(), ParameterType::INTEGER); + $qb->setParameter('updated_timestamp', $update_timestamp, ParameterType::INTEGER); $qb->where('uuid = :uuid'); $qb->setParameter('uuid', $client_uuid); $qb->setMaxResults(1); @@ -212,6 +260,34 @@ throw new ClientNotFoundException($client_uuid); } + // Update the record in redis if caching is enabled + if(Configuration::isCacheSystemEnabled() && Configuration::getObjectCacheEnabled('client_objects')) + { + try + { + $redis = RedisConnectionManager::getConnection( + Configuration::getObjectCacheServerPreference('client_objects'), + Configuration::getObjectCacheServerFallback('client_objects') + ); + + if($redis->exists($client_uuid)) + { + $redis->hSet($client_uuid, 'name', $name); + $redis->hSet($client_uuid, 'updated_timestamp', $update_timestamp); + if(Configuration::getObjectCacheTTL('client_objects') > 0) + { + $redis->expire($client_uuid, Configuration::getObjectCacheTTL('client_objects')); + } + + Log::debug(Misc::FEDERATIONLIB, sprintf('Updated client object %s <%s> in cache', $client_uuid, 'name')); + } + } + catch(Exception $e) + { + Log::warning(Misc::FEDERATIONLIB, sprintf('Failed to update client object %s in cache: %s', $client_uuid, $e->getMessage()), $e); + } + } + Log::verbose('net.nosial.federationlib', sprintf('Changed client name for client %s to %s', $client_uuid, $name)); } @@ -237,12 +313,13 @@ throw new InvalidClientDescriptionException(sprintf('Invalid client description: %s', $description)); } + $updated_timestamp = time(); $qb = Database::getConnection()->createQueryBuilder(); $qb->update(DatabaseTables::CLIENTS); $qb->set('description', ':description'); $qb->setParameter('description', $description, (is_null($description) ? ParameterType::NULL : ParameterType::STRING)); $qb->set('updated_timestamp', ':updated_timestamp'); - $qb->setParameter('updated_timestamp', time(), ParameterType::INTEGER); + $qb->setParameter('updated_timestamp', $updated_timestamp, ParameterType::INTEGER); $qb->where('uuid = :uuid'); $qb->setParameter('uuid', $client_uuid); $qb->setMaxResults(1); @@ -261,6 +338,35 @@ throw new ClientNotFoundException($client_uuid); } + if(Configuration::isCacheSystemEnabled() && Configuration::getObjectCacheEnabled('client_objects')) + { + try + { + $redis = RedisConnectionManager::getConnection( + Configuration::getObjectCacheServerPreference('client_objects'), + Configuration::getObjectCacheServerFallback('client_objects') + ); + + if($redis->exists($client_uuid)) + { + $redis->hSet($client_uuid, 'description', $description); + $redis->hSet($client_uuid, 'updated_timestamp', $updated_timestamp); + + if(Configuration::getObjectCacheTTL('client_objects') > 0) + { + $redis->expire($client_uuid, Configuration::getObjectCacheTTL('client_objects')); + } + + Log::debug(Misc::FEDERATIONLIB, sprintf('Updated client object %s <%s> in cache', $client_uuid, 'description')); + + } + } + catch(Exception $e) + { + Log::warning(Misc::FEDERATIONLIB, sprintf('Failed to update client object %s in cache: %s', $client_uuid, $e->getMessage()), $e); + } + } + Log::verbose('net.nosial.federationlib', sprintf('Changed client description for client %s to %s', $client_uuid, $description)); } @@ -286,14 +392,13 @@ throw new InvalidPermissionRoleException(sprintf('Invalid permission role: %s', $permission_role)); } - $time = time(); - + $updated_timestamp = time(); $qb = Database::getConnection()->createQueryBuilder(); $qb->update(DatabaseTables::CLIENTS); $qb->set('permission_role', ':permission_role'); $qb->setParameter('permission_role', $permission_role, ParameterType::INTEGER); $qb->set('updated_timestamp', ':updated_timestamp'); - $qb->setParameter('updated_timestamp', $time, ParameterType::INTEGER); + $qb->setParameter('updated_timestamp', $updated_timestamp, ParameterType::INTEGER); $qb->where('uuid = :uuid'); $qb->setParameter('uuid', $client_uuid); $qb->setMaxResults(1); @@ -312,130 +417,35 @@ throw new ClientNotFoundException($client_uuid); } - Log::verbose('net.nosial.federationlib', sprintf('Changed client permission role for client %s to %s', $client_uuid, $permission_role)); - } - - - /** - * Updates a client record in the database, if the client does not exist it will be created. - * This function is cache aware, if the client is cached it will only update the changed values. - * - * @param Client $client - * @return void - * @throws DatabaseException - */ - public function updateClient(Client $client): void - { - $cached_client = null; - - if(Configuration::isRedisCacheClientObjectsEnabled()) + if(Configuration::isCacheSystemEnabled() && Configuration::getObjectCacheEnabled('client_objects')) { try { - if(Redis::getConnection()?->exists(sprintf('Client<%s>', $client->getUuid()))) + $redis = RedisConnectionManager::getConnection( + Configuration::getObjectCacheServerPreference('client_objects'), + Configuration::getObjectCacheServerFallback('client_objects') + ); + + if($redis->exists($client_uuid)) { - $cached_client = Client::fromArray(Redis::getConnection()?->hGetAll(sprintf('Client<%s>', $client->getUuid()))); + $redis->hSet($client_uuid, 'permission_role', $permission_role); + $redis->hSet($client_uuid, 'updated_timestamp', $updated_timestamp); + + if(Configuration::getObjectCacheTTL('client_objects') > 0) + { + $redis->expire($client_uuid, Configuration::getObjectCacheTTL('client_objects')); + } + + Log::debug(Misc::FEDERATIONLIB, sprintf('Updated client object %s <%s> in cache', $client_uuid, 'permission_role')); } } catch(Exception $e) { - Log::warning('net.nosial.federationlib', sprintf('Failed to get Client from redis: %s', $e->getMessage())); + Log::warning(Misc::FEDERATIONLIB, sprintf('Failed to update client object %s in cache: %s', $client_uuid, $e->getMessage()), $e); } } - $qb = Database::getConnection()->createQueryBuilder(); - $qb->update(DatabaseTables::CLIENTS); - $qb->set('updated_timestamp', ':updated_timestamp'); - $qb->setParameter('updated_timestamp', time()); - $qb->where('uuid = :uuid'); - $qb->setParameter('uuid', $client->getUuid()); - - if($cached_client instanceof Client) - { - $data = array_diff($client->toArray(), $cached_client->toArray()); - } - else - { - $data = $client->toArray(); - } - - foreach($data as $key => $value) - { - switch($key) - { - case 'uuid': - case 'created_timestamp': - case 'updated_timestamp': - case 'seen_timestamp': - break; - - case 'name': - if($value === null || strlen($value) === 0 || !preg_match('/^[a-zA-Z0-9_\-]+$/', $value )) - { - break; - } - - $qb->set($key, ':' . $key); - $qb->setParameter($key, substr($value, 0, 64)); - break; - - case 'description': - if($value !== null) - { - $qb->set($key, ':' . $key); - $qb->setParameter($key, substr($value, 0, 255)); - } - break; - - case 'enabled': - $qb->set($key, ':' . $key); - $qb->setParameter($key, $value ? 1 : 0); - break; - - default: - $qb->set($key, ':' . $key); - $qb->setParameter($key, $value); - break; - } - } - - try - { - $qb->executeStatement(); - } - catch(Exception $e) - { - throw new DatabaseException('Failed to update client: ' . $e->getMessage(), $e); - } - - if(Configuration::isRedisCacheClientObjectsEnabled()) - { - // Update the differences in the cache - if($cached_client instanceof Client) - { - try - { - Redis::getConnection()?->hMSet((string)$client, array_diff($client->toArray(), $cached_client->toArray())); - Redis::getConnection()?->expire((string)$client, Configuration::getRedisCacheClientObjectsTTL()); - } - catch(Exception $e) - { - Log::warning('net.nosial.federationlib', sprintf('Failed to cache client in redis: %s', $e->getMessage())); - } - } - else - { - try - { - Redis::getConnection()?->hMSet((string)$client, $client->toArray()); - Redis::getConnection()?->expire((string)$client, $client->getUuid(), Configuration::getRedisCacheClientObjectsTTL()); - } - catch(Exception $e) - { - Log::warning('net.nosial.federationlib', sprintf('Failed to cache Client in redis: %s', $e->getMessage())); - } - } - } + Log::verbose('net.nosial.federationlib', sprintf('Changed client permission role for client %s to %s', $client_uuid, $permission_role)); } /** @@ -471,18 +481,34 @@ throw new DatabaseException('Failed to update last seen timestamp: ' . $e->getMessage(), $e); } - // Update the 'seen_timestamp' only in the hash table in redis - if(Configuration::isRedisCacheClientObjectsEnabled()) + if(Configuration::isCacheSystemEnabled() && Configuration::getObjectCacheEnabled('client_objects')) { try { - Redis::getConnection()?->hSet(sprintf('Client<%s>', $uuid), 'seen_timestamp', $timestamp); + $redis = RedisConnectionManager::getConnection( + Configuration::getObjectCacheServerPreference('client_objects'), + Configuration::getObjectCacheServerFallback('client_objects') + ); + + if($redis->exists($uuid)) + { + $redis->hSet($uuid, 'seen_timestamp', $timestamp); + + if(Configuration::getObjectCacheTTL('client_objects') > 0) + { + $redis->expire($uuid, Configuration::getObjectCacheTTL('client_objects')); + } + + Log::debug(Misc::FEDERATIONLIB, sprintf('Updated client object %s <%s> in cache', $uuid, 'seen_timestamp')); + } } catch(Exception $e) { - Log::warning('net.nosial.federationlib', sprintf('Failed to update last seen timestamp in redis: %s', $e->getMessage())); + Log::warning(Misc::FEDERATIONLIB, sprintf('Failed to update client object %s in cache: %s', $uuid, $e->getMessage()), $e); } } + + Log::verbose('net.nosial.federationlib', sprintf('Updated last seen timestamp for client %s to %s', $uuid, $timestamp)); } /** @@ -499,9 +525,10 @@ { $qb = Database::getConnection()->createQueryBuilder(); $qb->select( - 'uuid', 'enabled', 'name', 'description', 'secret_totp', 'query_permission', 'update_permission', + 'uuid', 'enabled', 'name', 'description', 'secret_totp', 'permission_role', 'flags', 'created_timestamp', 'updated_timestamp', 'seen_timestamp' ); + $qb->from(DatabaseTables::CLIENTS); $qb->setFirstResult(($page - 1) * $max_items); $qb->setMaxResults($max_items); @@ -511,6 +538,23 @@ throw new DatabaseException('Invalid order: ' . $order); } + $redis_client = null; + + if(Configuration::isCacheSystemEnabled() && Configuration::getObjectCacheEnabled('client_objects')) + { + try + { + $redis_client = RedisConnectionManager::getConnection( + Configuration::getObjectCacheServerPreference('client_objects'), + Configuration::getObjectCacheServerFallback('client_objects') + ); + } + catch(Exception $e) + { + Log::warning(Misc::FEDERATIONLIB, sprintf('Failed to connect to Redis server: %s', $e->getMessage()), $e); + } + } + switch($filter) { case ListClientsFilter::CREATED_TIMESTAMP: @@ -540,14 +584,27 @@ while($row = $result->fetchAssociative()) { - $clients[] = Client::fromArray($row); + $client_object = Client::fromArray($row); + + if($redis_client !== null) + { + $redis_client->hMSet($client_object->getUuid(), $client_object->toArray()); + + if(Configuration::getObjectCacheTTL('client_objects') > 0) + { + $redis_client->expire($row['uuid'], Configuration::getObjectCacheTTL('client_objects')); + } + } + + $clients[] = $client_object; } } catch(Exception $e) { - throw new DatabaseException('Failed to list clients: ' . $e->getMessage(), $e->getCode(), $e); + throw new DatabaseException('Failed to list clients: ' . $e->getMessage(), $e); } + unset($client); return $clients; } @@ -571,7 +628,7 @@ } catch(Exception $e) { - throw new DatabaseException('Failed to get total clients: ' . $e->getMessage(), $e->getCode(), $e); + throw new DatabaseException('Failed to get total clients: ' . $e->getMessage(), $e); } return (int)$row['COUNT(uuid)']; @@ -614,19 +671,26 @@ } catch(Exception $e) { - throw new DatabaseException('Failed to delete client: ' . $e->getMessage(), $e->getCode(), $e); + throw new DatabaseException('Failed to delete client: ' . $e->getMessage(), $e); } - // Invalidate the cache - if(Configuration::isRedisCacheClientObjectsEnabled()) + if(Configuration::isCacheSystemEnabled() && Configuration::getObjectCacheEnabled('client_objects')) { try { - Redis::getConnection()?->del(sprintf('Client<%s>', $uuid)); + $redis = RedisConnectionManager::getConnection( + Configuration::getObjectCacheServerPreference('client_objects'), + Configuration::getObjectCacheServerFallback('client_objects') + ); + + if($redis->exists($uuid)) + { + $redis->del($uuid); + } } catch(Exception $e) { - Log::warning('net.nosial.federationlib', sprintf('Failed to invalidate client cache in redis: %s', $e->getMessage())); + Log::warning(Misc::FEDERATIONLIB, sprintf('Failed to delete client object %s from cache: %s', $uuid, $e->getMessage()), $e); } } } diff --git a/src/FederationLib/Managers/RedisConnectionManager.php b/src/FederationLib/Managers/RedisConnectionManager.php index ed2c916..4b68169 100644 --- a/src/FederationLib/Managers/RedisConnectionManager.php +++ b/src/FederationLib/Managers/RedisConnectionManager.php @@ -29,7 +29,6 @@ * @param string|null $fallback * @return \Redis * @throws CacheConnectionException - * @throws CacheDriverException */ public static function getConnection(?string $name=null, ?string $fallback=null): \Redis {