From 264fff5c36e05e3f38cc6480a8ab031e62c4b286 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Julien=20Fastr=C3=A9?= <julien.fastre@champs-libres.coop>
Date: Fri, 24 Nov 2023 12:11:29 +0100
Subject: [PATCH] Implement asynchronous upload feature in Openstack Object
 Store

Those features were previously stored in champs-libres/async-upload-bundle

This commit introduces several new classes within the ChillDocStore bundle for handling asynchronous uploads onto an Openstack Object Store. Specifically, the TempUrlOpenstackGenerator, SignedUrl, TempUrlGeneratorInterface, TempUrlGenerateEvent, and TempUrlGeneratorException classes have been created.

This implementation will allow for generating "temporary URLs", which assist in securely and temporarily uploading resources to the OpenStack Object Store. This feature enables the handling of file uploads in a more scalable and efficient manner in high-volume environments.

Additionally, corresponding unit tests have also been added to ensure the accuracy of this new feature.
---
 .../TempUrlOpenstackGenerator.php             | 192 ++++++++++++++++++
 .../Event/TempUrlGenerateEvent.php            |  31 +++
 .../Exception/TempUrlGeneratorException.php   |  14 ++
 .../AsyncUpload/SignedUrl.php                 |  21 ++
 .../AsyncUpload/SignedUrlPost.php             |  28 +++
 .../AsyncUpload/TempUrlGeneratorInterface.php |  23 +++
 .../TempUrlOpenstackGeneratorTest.php         | 151 ++++++++++++++
 7 files changed, 460 insertions(+)
 create mode 100644 src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php
 create mode 100644 src/Bundle/ChillDocStoreBundle/AsyncUpload/Event/TempUrlGenerateEvent.php
 create mode 100644 src/Bundle/ChillDocStoreBundle/AsyncUpload/Exception/TempUrlGeneratorException.php
 create mode 100644 src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrl.php
 create mode 100644 src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrlPost.php
 create mode 100644 src/Bundle/ChillDocStoreBundle/AsyncUpload/TempUrlGeneratorInterface.php
 create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGeneratorTest.php

diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php
new file mode 100644
index 000000000..742e4285c
--- /dev/null
+++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php
@@ -0,0 +1,192 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\AsyncUpload\Driver\OpenstackObjectStore;
+
+use Chill\DocStoreBundle\AsyncUpload\Event\TempUrlGenerateEvent;
+use Chill\DocStoreBundle\AsyncUpload\Exception\TempUrlGeneratorException;
+use Chill\DocStoreBundle\AsyncUpload\SignedUrl;
+use Chill\DocStoreBundle\AsyncUpload\SignedUrlPost;
+use Chill\DocStoreBundle\AsyncUpload\TempUrlGeneratorInterface;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Clock\ClockInterface;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+
+/**
+ * Generate a temp url.
+ */
+final readonly class TempUrlOpenstackGenerator implements TempUrlGeneratorInterface
+{
+    private const CHARACTERS = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+
+    public function __construct(
+        private LoggerInterface $logger,
+        private EventDispatcherInterface $event_dispatcher,
+        private ClockInterface $clock,
+        private string $base_url,
+        private string $key,
+        private int $max_expire_delay,
+        private int $max_submit_delay,
+        private int $max_post_file_size,
+        private int $max_file_count = 1
+    ) {}
+
+    /**
+     * @throws TempUrlGeneratorException
+     */
+    public function generatePost(
+        int $expire_delay = null,
+        int $submit_delay = null,
+        int $max_file_count = 1,
+    ): SignedUrlPost {
+        $delay = $expire_delay ?? $this->max_expire_delay;
+        $submit_delay ??= $this->max_submit_delay;
+
+        if ($delay < 2) {
+            throw new TempUrlGeneratorException("The delay of {$delay} is too ".'short (<2 sec) to properly use this token');
+        }
+
+        if ($delay > $this->max_expire_delay) {
+            throw new TempUrlGeneratorException('The given delay is greater than the max delay authorized.');
+        }
+
+        if ($submit_delay < 15) {
+            throw new TempUrlGeneratorException("The submit delay of {$delay} is too ".'short (<15 sec) to properly use this token');
+        }
+
+        if ($submit_delay > $this->max_submit_delay) {
+            throw new TempUrlGeneratorException('The given submit delay is greater than the max submit delay authorized.');
+        }
+
+        if ($max_file_count > $this->max_file_count) {
+            throw new TempUrlGeneratorException('The number of files is greater than the authorized number of files');
+        }
+
+        $expires = $this->clock->now()->add(new \DateInterval('PT'.(string) $delay.'S'));
+
+        $object_name = $this->generateObjectName();
+
+        $g = new SignedUrlPost(
+            $url = $this->generateUrl($object_name),
+            $expires,
+            $this->max_post_file_size,
+            $max_file_count,
+            $submit_delay,
+            '',
+            $object_name,
+            $this->generateSignaturePost($url, $expires)
+        );
+
+        $this->event_dispatcher->dispatch(
+            new TempUrlGenerateEvent($g),
+        );
+
+        $this->logger->info(
+            'generate signature for url',
+            (array) $g
+        );
+
+        return $g;
+    }
+
+    /**
+     * Generate an absolute public url for a GET request on the object.
+     */
+    public function generate(string $method, string $object_name, int $expire_delay = null): SignedUrl
+    {
+        if ($expire_delay > $this->max_expire_delay) {
+            throw new TempUrlGeneratorException(sprintf('The expire delay (%d) is greater than the max_expire_delay (%d)', $expire_delay, $this->max_expire_delay));
+        }
+        $url = $this->generateUrl($object_name);
+
+        $expires = $this->clock->now()
+            ->add(new \DateInterval(sprintf('PT%dS', $expire_delay ?? $this->max_expire_delay)));
+        $args = [
+            'temp_url_sig' => $this->generateSignature($method, $url, $expires),
+            'temp_url_expires' => $expires->format('U'),
+        ];
+        $url = $url.'?'.\http_build_query($args);
+
+        $signature = new SignedUrl($method, $url, $expires);
+
+        $this->event_dispatcher->dispatch(
+            new TempUrlGenerateEvent($signature)
+        );
+
+        return $signature;
+    }
+
+    private function generateUrl($relative_path): string
+    {
+        return match (str_ends_with($this->base_url, '/')) {
+            true => $this->base_url.$relative_path,
+            false => $this->base_url.'/'.$relative_path
+        };
+    }
+
+    private function generateObjectName()
+    {
+        // inspiration from https://stackoverflow.com/a/4356295/1572236
+        $charactersLength = strlen(self::CHARACTERS);
+        $randomString = '';
+        for ($i = 0; $i < 21; ++$i) {
+            $randomString .= self::CHARACTERS[random_int(0, $charactersLength - 1)];
+        }
+
+        return $randomString;
+    }
+
+    private function generateSignaturePost($url, \DateTimeImmutable $expires)
+    {
+        $path = \parse_url((string) $url, PHP_URL_PATH);
+
+        $body = sprintf(
+            "%s\n%s\n%s\n%s\n%s",
+            $path,
+            '', // redirect is empty
+            (string) $this->max_post_file_size,
+            (string) $this->max_file_count,
+            $expires->format('U')
+        )
+        ;
+
+        $this->logger->debug(
+            'generate signature post',
+            ['url' => $body, 'method' => 'POST']
+        );
+
+        return \hash_hmac('sha512', $body, $this->key, false);
+    }
+
+    private function generateSignature($method, $url, \DateTimeImmutable $expires)
+    {
+        if ('POST' === $method) {
+            return $this->generateSignaturePost($url, $expires);
+        }
+
+        $path = \parse_url((string) $url, PHP_URL_PATH);
+
+        $body = sprintf(
+            "%s\n%s\n%s",
+            $method,
+            $expires->format('U'),
+            $path
+        )
+        ;
+
+        $this->logger->debug(
+            'generate signature GET',
+            ['url' => $body, 'method' => 'GET']
+        );
+
+        return \hash_hmac('sha512', $body, $this->key, false);
+    }
+}
diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/Event/TempUrlGenerateEvent.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Event/TempUrlGenerateEvent.php
new file mode 100644
index 000000000..619dba4e0
--- /dev/null
+++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Event/TempUrlGenerateEvent.php
@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\AsyncUpload\Event;
+
+use Chill\DocStoreBundle\AsyncUpload\SignedUrl;
+
+final class TempUrlGenerateEvent extends \Symfony\Contracts\EventDispatcher\Event
+{
+    final public const NAME_GENERATE = 'async_uploader.generate_url';
+
+    public function __construct(private readonly SignedUrl $data) {}
+
+    public function getMethod(): string
+    {
+        return $this->data->method;
+    }
+
+    public function getData(): SignedUrl
+    {
+        return $this->data;
+    }
+}
diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/Exception/TempUrlGeneratorException.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Exception/TempUrlGeneratorException.php
new file mode 100644
index 000000000..29c7e78ed
--- /dev/null
+++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Exception/TempUrlGeneratorException.php
@@ -0,0 +1,14 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\AsyncUpload\Exception;
+
+class TempUrlGeneratorException extends \RuntimeException {}
diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrl.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrl.php
new file mode 100644
index 000000000..583325a26
--- /dev/null
+++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrl.php
@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\AsyncUpload;
+
+readonly class SignedUrl
+{
+    public function __construct(
+        public string $method,
+        public string $url,
+        public \DateTimeImmutable $expires,
+    ) {}
+}
diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrlPost.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrlPost.php
new file mode 100644
index 000000000..98e273635
--- /dev/null
+++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/SignedUrlPost.php
@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\AsyncUpload;
+
+readonly class SignedUrlPost extends SignedUrl
+{
+    public function __construct(
+        string $url,
+        \DateTimeImmutable $expires,
+        public int $max_file_size,
+        public int $max_file_count,
+        public int $submit_delay,
+        public string $redirect,
+        public string $prefix,
+        public string $signature,
+    ) {
+        parent::__construct('POST', $url, $expires);
+    }
+}
diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/TempUrlGeneratorInterface.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/TempUrlGeneratorInterface.php
new file mode 100644
index 000000000..71be97ff9
--- /dev/null
+++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/TempUrlGeneratorInterface.php
@@ -0,0 +1,23 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\AsyncUpload;
+
+interface TempUrlGeneratorInterface
+{
+    public function generatePost(
+        int $expire_delay = null,
+        int $submit_delay = null,
+        int $max_file_count = 1
+    ): SignedUrlPost;
+
+    public function generate(string $method, string $object_name, int $expire_delay = null): SignedUrl;
+}
diff --git a/src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGeneratorTest.php b/src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGeneratorTest.php
new file mode 100644
index 000000000..25c5a1ebc
--- /dev/null
+++ b/src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGeneratorTest.php
@@ -0,0 +1,151 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace AsyncUpload\Driver\OpenstackObjectStore;
+
+use Chill\DocStoreBundle\AsyncUpload\Driver\OpenstackObjectStore\TempUrlOpenstackGenerator;
+use Chill\DocStoreBundle\AsyncUpload\SignedUrl;
+use Chill\DocStoreBundle\AsyncUpload\SignedUrlPost;
+use PHPUnit\Framework\TestCase;
+use Psr\Log\NullLogger;
+use Symfony\Component\Clock\MockClock;
+use Symfony\Component\EventDispatcher\EventDispatcher;
+
+/**
+ * @internal
+ *
+ * @coversNothing
+ */
+class TempUrlOpenstackGeneratorTest extends TestCase
+{
+    /**
+     * @dataProvider dataProviderGenerate
+     */
+    public function testGenerate(string $baseUrl, \DateTimeImmutable $now, string $key, string $method, string $objectName, int $expireDelay, SignedUrl $expected): void
+    {
+        $logger = new NullLogger();
+        $eventDispatcher = new EventDispatcher();
+        $clock = new MockClock($now);
+
+        $generator = new TempUrlOpenstackGenerator(
+            $logger,
+            $eventDispatcher,
+            $clock,
+            $baseUrl,
+            $key,
+            1800,
+            1800,
+            150
+        );
+
+        $signedUrl = $generator->generate($method, $objectName, $expireDelay);
+
+        self::assertEquals($expected, $signedUrl);
+    }
+
+    /**
+     * @dataProvider dataProviderGeneratePost
+     */
+    public function testGeneratePost(string $baseUrl, \DateTimeImmutable $now, string $key, string $method, string $objectName, int $expireDelay, SignedUrl $expected): void
+    {
+        $logger = new NullLogger();
+        $eventDispatcher = new EventDispatcher();
+        $clock = new MockClock($now);
+
+        $generator = new TempUrlOpenstackGenerator(
+            $logger,
+            $eventDispatcher,
+            $clock,
+            $baseUrl,
+            $key,
+            1800,
+            1800,
+            150
+        );
+
+        $signedUrl = $generator->generatePost();
+
+        self::assertEquals('POST', $signedUrl->method);
+        self::assertEquals((int) $clock->now()->format('U') + 1800, $signedUrl->expires->getTimestamp());
+        self::assertEquals(150, $signedUrl->max_file_size);
+        self::assertEquals(1, $signedUrl->max_file_count);
+        self::assertEquals(1800, $signedUrl->submit_delay);
+        self::assertEquals('', $signedUrl->redirect);
+        self::assertGreaterThanOrEqual(20, strlen($signedUrl->prefix));
+    }
+
+    public function dataProviderGenerate(): iterable
+    {
+        $now = \DateTimeImmutable::createFromFormat('U', '1702041743');
+        $expireDelay = 1800;
+        $baseUrls = [
+            'https://objectstore.example/v1/my_account/container/',
+            'https://objectstore.example/v1/my_account/container',
+        ];
+        $objectName = 'object';
+        $method = 'GET';
+        $key = 'MYKEY';
+
+        $signedUrl = new SignedUrl(
+            'GET',
+            'https://objectstore.example/v1/my_account/container/object?temp_url_sig=0aeef353a5f6e22d125c76c6ad8c644a59b222ba1b13eaeb56bf3d04e28b081d11dfcb36601ab3aa7b623d79e1ef03017071bbc842fb7b34afec2baff895bf80&temp_url_expires=1702043543',
+            \DateTimeImmutable::createFromFormat('U', '1702043543')
+        );
+
+        foreach ($baseUrls as $baseUrl) {
+            yield [
+                $baseUrl,
+                $now,
+                $key,
+                $method,
+                $objectName,
+                $expireDelay,
+                $signedUrl,
+            ];
+        }
+    }
+
+    public function dataProviderGeneratePost(): iterable
+    {
+        $now = \DateTimeImmutable::createFromFormat('U', '1702041743');
+        $expireDelay = 1800;
+        $baseUrls = [
+            'https://objectstore.example/v1/my_account/container/',
+            'https://objectstore.example/v1/my_account/container',
+        ];
+        $objectName = 'object';
+        $method = 'GET';
+        $key = 'MYKEY';
+
+        $signedUrl = new SignedUrlPost(
+            'https://objectstore.example/v1/my_account/container/object?temp_url_sig=0aeef353a5f6e22d125c76c6ad8c644a59b222ba1b13eaeb56bf3d04e28b081d11dfcb36601ab3aa7b623d79e1ef03017071bbc842fb7b34afec2baff895bf80&temp_url_expires=1702043543',
+            \DateTimeImmutable::createFromFormat('U', '1702043543'),
+            150,
+            1,
+            1800,
+            '',
+            'abc',
+            'abc'
+        );
+
+        foreach ($baseUrls as $baseUrl) {
+            yield [
+                $baseUrl,
+                $now,
+                $key,
+                $method,
+                $objectName,
+                $expireDelay,
+                $signedUrl,
+            ];
+        }
+    }
+}