mirror of
https://gitlab.com/Chill-Projet/chill-bundles.git
synced 2025-06-07 18:44:08 +00:00
Implement asynchronous upload feature in Openstack Object Store
Those features were previously stored in champs-libres/async-upload-bundle This commit introduces several new classes within the ChillDocStore bundle for handling asynchronous uploads onto an Openstack Object Store. Specifically, the TempUrlOpenstackGenerator, SignedUrl, TempUrlGeneratorInterface, TempUrlGenerateEvent, and TempUrlGeneratorException classes have been created. This implementation will allow for generating "temporary URLs", which assist in securely and temporarily uploading resources to the OpenStack Object Store. This feature enables the handling of file uploads in a more scalable and efficient manner in high-volume environments. Additionally, corresponding unit tests have also been added to ensure the accuracy of this new feature.
This commit is contained in:
parent
91e6b035bd
commit
264fff5c36
GPG Key ID:
BDE2190974723FCB
@ -0,0 +1,192 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/*
|
||||
* Chill is a software for social workers
|
||||
*
|
||||
* For the full copyright and license information, please view
|
||||
* the LICENSE file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Chill\DocStoreBundle\AsyncUpload\Driver\OpenstackObjectStore;
|
||||
|
||||
use Chill\DocStoreBundle\AsyncUpload\Event\TempUrlGenerateEvent;
|
||||
use Chill\DocStoreBundle\AsyncUpload\Exception\TempUrlGeneratorException;
|
||||
use Chill\DocStoreBundle\AsyncUpload\SignedUrl;
|
||||
use Chill\DocStoreBundle\AsyncUpload\SignedUrlPost;
|
||||
use Chill\DocStoreBundle\AsyncUpload\TempUrlGeneratorInterface;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Clock\ClockInterface;
|
||||
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
|
||||
|
||||
/**
|
||||
* Generate a temp url.
|
||||
*/
|
||||
final readonly class TempUrlOpenstackGenerator implements TempUrlGeneratorInterface
|
||||
{
|
||||
private const CHARACTERS = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
|
||||
|
||||
public function __construct(
|
||||
private LoggerInterface $logger,
|
||||
private EventDispatcherInterface $event_dispatcher,
|
||||
private ClockInterface $clock,
|
||||
private string $base_url,
|
||||
private string $key,
|
||||
private int $max_expire_delay,
|
||||
private int $max_submit_delay,
|
||||
private int $max_post_file_size,
|
||||
private int $max_file_count = 1
|
||||
) {}
|
||||
|
||||
/**
|
||||
* @throws TempUrlGeneratorException
|
||||
*/
|
||||
public function generatePost(
|
||||
int $expire_delay = null,
|
||||
int $submit_delay = null,
|
||||
int $max_file_count = 1,
|
||||
): SignedUrlPost {
|
||||
$delay = $expire_delay ?? $this->max_expire_delay;
|
||||
$submit_delay ??= $this->max_submit_delay;
|
||||
|
||||
if ($delay < 2) {
|
||||
throw new TempUrlGeneratorException("The delay of {$delay} is too ".'short (<2 sec) to properly use this token');
|
||||
}
|
||||
|
||||
if ($delay > $this->max_expire_delay) {
|
||||
throw new TempUrlGeneratorException('The given delay is greater than the max delay authorized.');
|
||||
}
|
||||
|
||||
if ($submit_delay < 15) {
|
||||
throw new TempUrlGeneratorException("The submit delay of {$delay} is too ".'short (<15 sec) to properly use this token');
|
||||
}
|
||||
|
||||
if ($submit_delay > $this->max_submit_delay) {
|
||||
throw new TempUrlGeneratorException('The given submit delay is greater than the max submit delay authorized.');
|
||||
}
|
||||
|
||||
if ($max_file_count > $this->max_file_count) {
|
||||
throw new TempUrlGeneratorException('The number of files is greater than the authorized number of files');
|
||||
}
|
||||
|
||||
$expires = $this->clock->now()->add(new \DateInterval('PT'.(string) $delay.'S'));
|
||||
|
||||
$object_name = $this->generateObjectName();
|
||||
|
||||
$g = new SignedUrlPost(
|
||||
$url = $this->generateUrl($object_name),
|
||||
$expires,
|
||||
$this->max_post_file_size,
|
||||
$max_file_count,
|
||||
$submit_delay,
|
||||
'',
|
||||
$object_name,
|
||||
$this->generateSignaturePost($url, $expires)
|
||||
);
|
||||
|
||||
$this->event_dispatcher->dispatch(
|
||||
new TempUrlGenerateEvent($g),
|
||||
);
|
||||
|
||||
$this->logger->info(
|
||||
'generate signature for url',
|
||||
(array) $g
|
||||
);
|
||||
|
||||
return $g;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an absolute public url for a GET request on the object.
|
||||
*/
|
||||
public function generate(string $method, string $object_name, int $expire_delay = null): SignedUrl
|
||||
{
|
||||
if ($expire_delay > $this->max_expire_delay) {
|
||||
throw new TempUrlGeneratorException(sprintf('The expire delay (%d) is greater than the max_expire_delay (%d)', $expire_delay, $this->max_expire_delay));
|
||||
}
|
||||
$url = $this->generateUrl($object_name);
|
||||
|
||||
$expires = $this->clock->now()
|
||||
->add(new \DateInterval(sprintf('PT%dS', $expire_delay ?? $this->max_expire_delay)));
|
||||
$args = [
|
||||
'temp_url_sig' => $this->generateSignature($method, $url, $expires),
|
||||
'temp_url_expires' => $expires->format('U'),
|
||||
];
|
||||
$url = $url.'?'.\http_build_query($args);
|
||||
|
||||
$signature = new SignedUrl($method, $url, $expires);
|
||||
|
||||
$this->event_dispatcher->dispatch(
|
||||
new TempUrlGenerateEvent($signature)
|
||||
);
|
||||
|
||||
return $signature;
|
||||
}
|
||||
|
||||
private function generateUrl($relative_path): string
|
||||
{
|
||||
return match (str_ends_with($this->base_url, '/')) {
|
||||
true => $this->base_url.$relative_path,
|
||||
false => $this->base_url.'/'.$relative_path
|
||||
};
|
||||
}
|
||||
|
||||
private function generateObjectName()
|
||||
{
|
||||
// inspiration from https://stackoverflow.com/a/4356295/1572236
|
||||
$charactersLength = strlen(self::CHARACTERS);
|
||||
$randomString = '';
|
||||
for ($i = 0; $i < 21; ++$i) {
|
||||
$randomString .= self::CHARACTERS[random_int(0, $charactersLength - 1)];
|
||||
}
|
||||
|
||||
return $randomString;
|
||||
}
|
||||
|
||||
private function generateSignaturePost($url, \DateTimeImmutable $expires)
|
||||
{
|
||||
$path = \parse_url((string) $url, PHP_URL_PATH);
|
||||
|
||||
$body = sprintf(
|
||||
"%s\n%s\n%s\n%s\n%s",
|
||||
$path,
|
||||
'', // redirect is empty
|
||||
(string) $this->max_post_file_size,
|
||||
(string) $this->max_file_count,
|
||||
$expires->format('U')
|
||||
)
|
||||
;
|
||||
|
||||
$this->logger->debug(
|
||||
'generate signature post',
|
||||
['url' => $body, 'method' => 'POST']
|
||||
);
|
||||
|
||||
return \hash_hmac('sha512', $body, $this->key, false);
|
||||
}
|
||||
|
||||
private function generateSignature($method, $url, \DateTimeImmutable $expires)
|
||||
{
|
||||
if ('POST' === $method) {
|
||||
return $this->generateSignaturePost($url, $expires);
|
||||
}
|
||||
|
||||
$path = \parse_url((string) $url, PHP_URL_PATH);
|
||||
|
||||
$body = sprintf(
|
||||
"%s\n%s\n%s",
|
||||
$method,
|
||||
$expires->format('U'),
|
||||
$path
|
||||
)
|
||||
;
|
||||
|
||||
$this->logger->debug(
|
||||
'generate signature GET',
|
||||
['url' => $body, 'method' => 'GET']
|
||||
);
|
||||
|
||||
return \hash_hmac('sha512', $body, $this->key, false);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/*
|
||||
* Chill is a software for social workers
|
||||
*
|
||||
* For the full copyright and license information, please view
|
||||
* the LICENSE file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Chill\DocStoreBundle\AsyncUpload\Event;
|
||||
|
||||
use Chill\DocStoreBundle\AsyncUpload\SignedUrl;
|
||||
|
||||
final class TempUrlGenerateEvent extends \Symfony\Contracts\EventDispatcher\Event
|
||||
{
|
||||
final public const NAME_GENERATE = 'async_uploader.generate_url';
|
||||
|
||||
public function __construct(private readonly SignedUrl $data) {}
|
||||
|
||||
public function getMethod(): string
|
||||
{
|
||||
return $this->data->method;
|
||||
}
|
||||
|
||||
public function getData(): SignedUrl
|
||||
{
|
||||
return $this->data;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/*
|
||||
* Chill is a software for social workers
|
||||
*
|
||||
* For the full copyright and license information, please view
|
||||
* the LICENSE file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Chill\DocStoreBundle\AsyncUpload\Exception;
|
||||
|
||||
class TempUrlGeneratorException extends \RuntimeException {}
|
||||
21
src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrl.php
Normal file
21
src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrl.php
Normal file
@ -0,0 +1,21 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/*
|
||||
* Chill is a software for social workers
|
||||
*
|
||||
* For the full copyright and license information, please view
|
||||
* the LICENSE file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Chill\DocStoreBundle\AsyncUpload;
|
||||
|
||||
readonly class SignedUrl
|
||||
{
|
||||
public function __construct(
|
||||
public string $method,
|
||||
public string $url,
|
||||
public \DateTimeImmutable $expires,
|
||||
) {}
|
||||
}
|
||||
28
src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrlPost.php
Normal file
28
src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrlPost.php
Normal file
@ -0,0 +1,28 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/*
|
||||
* Chill is a software for social workers
|
||||
*
|
||||
* For the full copyright and license information, please view
|
||||
* the LICENSE file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Chill\DocStoreBundle\AsyncUpload;
|
||||
|
||||
readonly class SignedUrlPost extends SignedUrl
|
||||
{
|
||||
public function __construct(
|
||||
string $url,
|
||||
\DateTimeImmutable $expires,
|
||||
public int $max_file_size,
|
||||
public int $max_file_count,
|
||||
public int $submit_delay,
|
||||
public string $redirect,
|
||||
public string $prefix,
|
||||
public string $signature,
|
||||
) {
|
||||
parent::__construct('POST', $url, $expires);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/*
|
||||
* Chill is a software for social workers
|
||||
*
|
||||
* For the full copyright and license information, please view
|
||||
* the LICENSE file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Chill\DocStoreBundle\AsyncUpload;
|
||||
|
||||
interface TempUrlGeneratorInterface
|
||||
{
|
||||
public function generatePost(
|
||||
int $expire_delay = null,
|
||||
int $submit_delay = null,
|
||||
int $max_file_count = 1
|
||||
): SignedUrlPost;
|
||||
|
||||
public function generate(string $method, string $object_name, int $expire_delay = null): SignedUrl;
|
||||
}
|
||||
@ -0,0 +1,151 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/*
|
||||
* Chill is a software for social workers
|
||||
*
|
||||
* For the full copyright and license information, please view
|
||||
* the LICENSE file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace AsyncUpload\Driver\OpenstackObjectStore;
|
||||
|
||||
use Chill\DocStoreBundle\AsyncUpload\Driver\OpenstackObjectStore\TempUrlOpenstackGenerator;
|
||||
use Chill\DocStoreBundle\AsyncUpload\SignedUrl;
|
||||
use Chill\DocStoreBundle\AsyncUpload\SignedUrlPost;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Psr\Log\NullLogger;
|
||||
use Symfony\Component\Clock\MockClock;
|
||||
use Symfony\Component\EventDispatcher\EventDispatcher;
|
||||
|
||||
/**
|
||||
* @internal
|
||||
*
|
||||
* @coversNothing
|
||||
*/
|
||||
class TempUrlOpenstackGeneratorTest extends TestCase
|
||||
{
|
||||
/**
|
||||
* @dataProvider dataProviderGenerate
|
||||
*/
|
||||
public function testGenerate(string $baseUrl, \DateTimeImmutable $now, string $key, string $method, string $objectName, int $expireDelay, SignedUrl $expected): void
|
||||
{
|
||||
$logger = new NullLogger();
|
||||
$eventDispatcher = new EventDispatcher();
|
||||
$clock = new MockClock($now);
|
||||
|
||||
$generator = new TempUrlOpenstackGenerator(
|
||||
$logger,
|
||||
$eventDispatcher,
|
||||
$clock,
|
||||
$baseUrl,
|
||||
$key,
|
||||
1800,
|
||||
1800,
|
||||
150
|
||||
);
|
||||
|
||||
$signedUrl = $generator->generate($method, $objectName, $expireDelay);
|
||||
|
||||
self::assertEquals($expected, $signedUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* @dataProvider dataProviderGeneratePost
|
||||
*/
|
||||
public function testGeneratePost(string $baseUrl, \DateTimeImmutable $now, string $key, string $method, string $objectName, int $expireDelay, SignedUrl $expected): void
|
||||
{
|
||||
$logger = new NullLogger();
|
||||
$eventDispatcher = new EventDispatcher();
|
||||
$clock = new MockClock($now);
|
||||
|
||||
$generator = new TempUrlOpenstackGenerator(
|
||||
$logger,
|
||||
$eventDispatcher,
|
||||
$clock,
|
||||
$baseUrl,
|
||||
$key,
|
||||
1800,
|
||||
1800,
|
||||
150
|
||||
);
|
||||
|
||||
$signedUrl = $generator->generatePost();
|
||||
|
||||
self::assertEquals('POST', $signedUrl->method);
|
||||
self::assertEquals((int) $clock->now()->format('U') + 1800, $signedUrl->expires->getTimestamp());
|
||||
self::assertEquals(150, $signedUrl->max_file_size);
|
||||
self::assertEquals(1, $signedUrl->max_file_count);
|
||||
self::assertEquals(1800, $signedUrl->submit_delay);
|
||||
self::assertEquals('', $signedUrl->redirect);
|
||||
self::assertGreaterThanOrEqual(20, strlen($signedUrl->prefix));
|
||||
}
|
||||
|
||||
public function dataProviderGenerate(): iterable
|
||||
{
|
||||
$now = \DateTimeImmutable::createFromFormat('U', '1702041743');
|
||||
$expireDelay = 1800;
|
||||
$baseUrls = [
|
||||
'https://objectstore.example/v1/my_account/container/',
|
||||
'https://objectstore.example/v1/my_account/container',
|
||||
];
|
||||
$objectName = 'object';
|
||||
$method = 'GET';
|
||||
$key = 'MYKEY';
|
||||
|
||||
$signedUrl = new SignedUrl(
|
||||
'GET',
|
||||
'https://objectstore.example/v1/my_account/container/object?temp_url_sig=0aeef353a5f6e22d125c76c6ad8c644a59b222ba1b13eaeb56bf3d04e28b081d11dfcb36601ab3aa7b623d79e1ef03017071bbc842fb7b34afec2baff895bf80&temp_url_expires=1702043543',
|
||||
\DateTimeImmutable::createFromFormat('U', '1702043543')
|
||||
);
|
||||
|
||||
foreach ($baseUrls as $baseUrl) {
|
||||
yield [
|
||||
$baseUrl,
|
||||
$now,
|
||||
$key,
|
||||
$method,
|
||||
$objectName,
|
||||
$expireDelay,
|
||||
$signedUrl,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
public function dataProviderGeneratePost(): iterable
|
||||
{
|
||||
$now = \DateTimeImmutable::createFromFormat('U', '1702041743');
|
||||
$expireDelay = 1800;
|
||||
$baseUrls = [
|
||||
'https://objectstore.example/v1/my_account/container/',
|
||||
'https://objectstore.example/v1/my_account/container',
|
||||
];
|
||||
$objectName = 'object';
|
||||
$method = 'GET';
|
||||
$key = 'MYKEY';
|
||||
|
||||
$signedUrl = new SignedUrlPost(
|
||||
'https://objectstore.example/v1/my_account/container/object?temp_url_sig=0aeef353a5f6e22d125c76c6ad8c644a59b222ba1b13eaeb56bf3d04e28b081d11dfcb36601ab3aa7b623d79e1ef03017071bbc842fb7b34afec2baff895bf80&temp_url_expires=1702043543',
|
||||
\DateTimeImmutable::createFromFormat('U', '1702043543'),
|
||||
150,
|
||||
1,
|
||||
1800,
|
||||
'',
|
||||
'abc',
|
||||
'abc'
|
||||
);
|
||||
|
||||
foreach ($baseUrls as $baseUrl) {
|
||||
yield [
|
||||
$baseUrl,
|
||||
$now,
|
||||
$key,
|
||||
$method,
|
||||
$objectName,
|
||||
$expireDelay,
|
||||
$signedUrl,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user