From 2b7ea4178b559f21ffd442ab6b170aa30d036f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Tue, 9 Jul 2024 22:23:24 +0200 Subject: [PATCH 01/25] Add versioning to stored objects This update introduces a versioning system to the stored objects in the ChillDocStoreBundle. The 'StoredObject' entity now includes several new methods, and maintains a collection of 'StoredObjectVersion' instances. Each time a 'StoredObject' is modified, a new version instance is created and added to the collection, ensuring a history of changes. Migration file for the addition of new database column included. Corresponding tests are also updated. --- .../Entity/StoredObject.php | 139 +++++++++++------- .../Entity/StoredObjectVersion.php | 127 ++++++++++++++++ .../Tests/Entity/StoredObjectTest.php | 49 +++++- .../migrations/Version20240709102730.php | 80 ++++++++++ 4 files changed, 333 insertions(+), 62 deletions(-) create mode 100644 src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php create mode 100644 src/Bundle/ChillDocStoreBundle/migrations/Version20240709102730.php diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php index 4472d2177..f37829a26 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php @@ -16,9 +16,12 @@ use ChampsLibres\WopiLib\Contract\Entity\Document; use Chill\DocGeneratorBundle\Entity\DocGeneratorTemplate; use Chill\MainBundle\Doctrine\Model\TrackCreationInterface; use Chill\MainBundle\Doctrine\Model\TrackCreationTrait; +use Doctrine\Common\Collections\ArrayCollection; +use Doctrine\Common\Collections\Collection; use Doctrine\ORM\Mapping as ORM; use Ramsey\Uuid\Uuid; use Ramsey\Uuid\UuidInterface; +use Random\RandomException; use Symfony\Component\Serializer\Annotation as Serializer; /** @@ -28,9 +31,12 @@ use Symfony\Component\Serializer\Annotation as Serializer; * * The property `$deleteAt` allow a deletion of the document after the given date. But this property should * be set before the document is actually written by the StoredObjectManager. + * + * Each version is stored within a @see{StoredObjectVersion}, associated with this current's object. The creation + * of each new version should be done using the method @see{self::registerVersion}. */ #[ORM\Entity] -#[ORM\Table('chill_doc.stored_object')] +#[ORM\Table('stored_object', schema: 'chill_doc')] #[AsyncFileExists(message: 'The file is not stored properly')] class StoredObject implements Document, TrackCreationInterface { @@ -43,9 +49,11 @@ class StoredObject implements Document, TrackCreationInterface #[ORM\Column(type: \Doctrine\DBAL\Types\Types::JSON, name: 'datas')] private array $datas = []; - #[Serializer\Groups(['write'])] - #[ORM\Column(type: \Doctrine\DBAL\Types\Types::TEXT)] - private string $filename = ''; + /** + * the prefix of each version. + */ + #[ORM\Column(type: \Doctrine\DBAL\Types\Types::TEXT, nullable: false, options: ['default' => ''])] + private string $prefix = ''; #[Serializer\Groups(['write'])] #[ORM\Id] @@ -53,25 +61,10 @@ class StoredObject implements Document, TrackCreationInterface #[ORM\Column(type: \Doctrine\DBAL\Types\Types::INTEGER)] private ?int $id = null; - /** - * @var int[] - */ #[Serializer\Groups(['write'])] - #[ORM\Column(type: \Doctrine\DBAL\Types\Types::JSON, name: 'iv')] - private array $iv = []; - - #[Serializer\Groups(['write'])] - #[ORM\Column(type: \Doctrine\DBAL\Types\Types::JSON, name: 'key')] - private array $keyInfos = []; - - #[Serializer\Groups(['write'])] - #[ORM\Column(type: \Doctrine\DBAL\Types\Types::TEXT, name: 'title')] + #[ORM\Column(name: 'title', type: \Doctrine\DBAL\Types\Types::TEXT, options: ['default' => ''])] private string $title = ''; - #[Serializer\Groups(['write'])] - #[ORM\Column(type: \Doctrine\DBAL\Types\Types::TEXT, name: 'type', options: ['default' => ''])] - private string $type = ''; - #[Serializer\Groups(['write'])] #[ORM\Column(type: 'uuid', unique: true)] private UuidInterface $uuid; @@ -94,6 +87,12 @@ class StoredObject implements Document, TrackCreationInterface #[ORM\Column(type: \Doctrine\DBAL\Types\Types::TEXT, nullable: false, options: ['default' => ''])] private string $generationErrors = ''; + /** + * @var Collection + */ + #[ORM\OneToMany(targetEntity: StoredObjectVersion::class, cascade: ['persist'], mappedBy: 'storedObject')] + private Collection $versions; + /** * @param StoredObject::STATUS_* $status */ @@ -102,6 +101,8 @@ class StoredObject implements Document, TrackCreationInterface private string $status = 'ready' ) { $this->uuid = Uuid::uuid4(); + $this->versions = new ArrayCollection(); + $this->prefix = self::generatePrefix(); } public function addGenerationTrial(): self @@ -125,14 +126,32 @@ class StoredObject implements Document, TrackCreationInterface return \DateTime::createFromImmutable($this->createdAt); } + public function getCurrentVersion(): ?StoredObjectVersion + { + $maxVersion = null; + + foreach ($this->versions as $v) { + if ($v->getVersion() > ($maxVersion?->getVersion() ?? -1)) { + $maxVersion = $v; + } + } + + return $maxVersion; + } + public function getDatas(): array { return $this->datas; } + public function getPrefix(): string + { + return $this->prefix; + } + public function getFilename(): string { - return $this->filename; + return $this->getCurrentVersion()?->getFilename() ?? ''; } public function getGenerationTrialsCounter(): int @@ -145,14 +164,17 @@ class StoredObject implements Document, TrackCreationInterface return $this->id; } + /** + * @return list + */ public function getIv(): array { - return $this->iv; + return $this->getCurrentVersion()?->getIv() ?? []; } public function getKeyInfos(): array { - return $this->keyInfos; + return $this->getCurrentVersion()?->getKeyInfos() ?? []; } /** @@ -171,14 +193,14 @@ class StoredObject implements Document, TrackCreationInterface return $this->status; } - public function getTitle() + public function getTitle(): string { return $this->title; } - public function getType() + public function getType(): string { - return $this->type; + return $this->getCurrentVersion()?->getType() ?? ''; } public function getUuid(): UuidInterface @@ -209,27 +231,6 @@ class StoredObject implements Document, TrackCreationInterface return $this; } - public function setFilename(?string $filename): self - { - $this->filename = (string) $filename; - - return $this; - } - - public function setIv(?array $iv): self - { - $this->iv = (array) $iv; - - return $this; - } - - public function setKeyInfos(?array $keyInfos): self - { - $this->keyInfos = (array) $keyInfos; - - return $this; - } - /** * @param StoredObject::STATUS_* $status */ @@ -247,18 +248,16 @@ class StoredObject implements Document, TrackCreationInterface return $this; } - public function setType(?string $type): self - { - $this->type = (string) $type; - - return $this; - } - public function getTemplate(): ?DocGeneratorTemplate { return $this->template; } + public function getVersions(): Collection + { + return $this->versions; + } + public function hasTemplate(): bool { return null !== $this->template; @@ -314,6 +313,29 @@ class StoredObject implements Document, TrackCreationInterface return $this; } + public function registerVersion( + array $iv = [], + array $keyInfos = [], + string $type = '', + ?string $filename = null, + ): StoredObjectVersion { + $version = new StoredObjectVersion( + $this, + null === $this->getCurrentVersion() ? 0 : $this->getCurrentVersion()->getVersion() + 1, + $iv, + $keyInfos, + $type, + $filename + ); + + $this->versions->add($version); + + return $version; + } + + /** + * @deprecated + */ public function saveHistory(): void { if ('' === $this->getFilename()) { @@ -328,4 +350,13 @@ class StoredObject implements Document, TrackCreationInterface 'before' => (new \DateTimeImmutable('now'))->getTimestamp(), ]; } + + public static function generatePrefix(): string + { + try { + return base_convert(bin2hex(random_bytes(8)), 16, 36); + } catch (RandomException $e) { + return uniqid(more_entropy: true); + } + } } diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php new file mode 100644 index 000000000..c280b30a3 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php @@ -0,0 +1,127 @@ + ''])] + private string $filename = ''; + + public function __construct( + /** + * The stored object associated with this version. + */ + #[ORM\ManyToOne(targetEntity: StoredObject::class, inversedBy: 'versions')] + #[ORM\JoinColumn(name: 'stored_object_id', nullable: true)] + private StoredObject $storedObject, + + /** + * The incremental version. + */ + #[ORM\Column(name: 'version', type: \Doctrine\DBAL\Types\Types::INTEGER, options: ['default' => 0])] + private int $version = 0, + + /** + * vector for encryption. + * + * @var int[] + */ + #[ORM\Column(type: \Doctrine\DBAL\Types\Types::JSON, name: 'iv')] + private array $iv = [], + + /** + * Key infos for document encryption. + * + * @var array + */ + #[ORM\Column(type: \Doctrine\DBAL\Types\Types::JSON, name: 'key')] + private array $keyInfos = [], + + /** + * type of the document. + */ + #[ORM\Column(type: \Doctrine\DBAL\Types\Types::TEXT, name: 'type', options: ['default' => ''])] + private string $type = '', + ?string $filename = null, + ) { + $this->filename = $filename ?? self::generateFilename($this); + } + + public static function generateFilename(StoredObjectVersion $storedObjectVersion): string + { + try { + $suffix = base_convert(bin2hex(random_bytes(16)), 16, 36); + } catch (RandomException $e) { + $suffix = uniqid(more_entropy: true); + } + + return $storedObjectVersion->getStoredObject()->getPrefix().'/'.$suffix; + } + + public function getFilename(): string + { + return $this->filename; + } + + public function getId(): ?int + { + return $this->id; + } + + public function getIv(): array + { + return $this->iv; + } + + public function getKeyInfos(): array + { + return $this->keyInfos; + } + + public function getStoredObject(): StoredObject + { + return $this->storedObject; + } + + public function getType(): string + { + return $this->type; + } + + public function getVersion(): int + { + return $this->version; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Entity/StoredObjectTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Entity/StoredObjectTest.php index 33394114f..9351947a4 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Entity/StoredObjectTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Entity/StoredObjectTest.php @@ -25,18 +25,22 @@ class StoredObjectTest extends KernelTestCase { $storedObject = new StoredObject(); $storedObject - ->setFilename('test_0') - ->setIv([2, 4, 6, 8]) - ->setKeyInfos(['key' => ['data0' => 'data0']]) - ->setType('text/html'); + ->registerVersion( + [2, 4, 6, 8], + ['key' => ['data0' => 'data0']], + 'text/html', + 'test_0', + ); $storedObject->saveHistory(); $storedObject - ->setFilename('test_1') - ->setIv([8, 10, 12]) - ->setKeyInfos(['key' => ['data1' => 'data1']]) - ->setType('text/text'); + ->registerVersion( + [8, 10, 12], + ['key' => ['data1' => 'data1']], + 'text/text', + 'test_1', + ); $storedObject->saveHistory(); @@ -50,4 +54,33 @@ class StoredObjectTest extends KernelTestCase self::assertEquals(['key' => ['data1' => 'data1']], $storedObject->getDatas()['history'][1]['key_infos']); self::assertEquals('text/text', $storedObject->getDatas()['history'][1]['type']); } + + public function testRegisterVersion(): void + { + $object = new StoredObject(); + $firstVersion = $object->registerVersion( + [5, 6, 7, 8], + ['key' => ['some key']], + 'text/html', + ); + $version = $object->registerVersion( + [1, 2, 3, 4], + $k = ['key' => ['data0' => 'data0']], + 'text/text', + 'abcde', + ); + + self::assertCount(2, $object->getVersions()); + self::assertEquals('abcde', $object->getFilename()); + self::assertEquals([1, 2, 3, 4], $object->getIv()); + self::assertEqualsCanonicalizing($k, $object->getKeyInfos()); + self::assertEquals('text/text', $object->getType()); + + self::assertEquals('abcde', $version->getFilename()); + self::assertEquals([1, 2, 3, 4], $version->getIv()); + self::assertEqualsCanonicalizing($k, $version->getKeyInfos()); + self::assertEquals('text/text', $version->getType()); + + self::assertNotSame($firstVersion, $version); + } } diff --git a/src/Bundle/ChillDocStoreBundle/migrations/Version20240709102730.php b/src/Bundle/ChillDocStoreBundle/migrations/Version20240709102730.php new file mode 100644 index 000000000..4ac01b196 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/migrations/Version20240709102730.php @@ -0,0 +1,80 @@ +addSql('CREATE SEQUENCE chill_doc.stored_object_version_id_seq INCREMENT BY 1 MINVALUE 1 START 1'); + $this->addSql( + <<<'SQL' + CREATE TABLE chill_doc.stored_object_version ( + id INT NOT NULL, + stored_object_id INT DEFAULT NULL, + version INT DEFAULT 0 NOT NULL, + filename TEXT NOT NULL, + iv JSON NOT NULL, + key JSON NOT NULL, + type TEXT DEFAULT '' NOT NULL, + createdAt TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL, + createdBy_id INT DEFAULT NULL, + PRIMARY KEY(id)) + SQL + ); + $this->addSql('CREATE INDEX IDX_C1D55302232D562B ON chill_doc.stored_object_version (stored_object_id)'); + $this->addSql('CREATE INDEX IDX_C1D553023174800F ON chill_doc.stored_object_version (createdBy_id)'); + $this->addSql('CREATE UNIQUE INDEX chill_doc_stored_object_version_unique_by_object ON chill_doc.stored_object_version (stored_object_id, version)'); + $this->addSql('COMMENT ON COLUMN chill_doc.stored_object_version.createdAt IS \'(DC2Type:datetime_immutable)\''); + $this->addSql('ALTER TABLE chill_doc.stored_object_version ADD CONSTRAINT FK_C1D55302232D562B FOREIGN KEY (stored_object_id) REFERENCES chill_doc.stored_object (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE chill_doc.stored_object_version ADD CONSTRAINT FK_C1D553023174800F FOREIGN KEY (createdBy_id) REFERENCES users (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql( + <<<'SQL' + INSERT INTO chill_doc.stored_object_version (id, stored_object_id, version, filename, iv, key, type) + SELECT nextval('chill_doc.stored_object_version_id_seq'), id, 1, filename, iv, key, type FROM chill_doc.stored_object + SQL + ); + $this->addSql('ALTER TABLE chill_doc.stored_object RENAME COLUMN filename TO prefix'); + $this->addSql('ALTER TABLE chill_doc.stored_object DROP key'); + $this->addSql('ALTER TABLE chill_doc.stored_object DROP iv'); + $this->addSql('ALTER TABLE chill_doc.stored_object DROP type'); + $this->addSql('ALTER TABLE chill_doc.stored_object ALTER title SET DEFAULT \'\''); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP SEQUENCE chill_doc.stored_object_version_id_seq CASCADE'); + $this->addSql('ALTER TABLE chill_doc.stored_object RENAME COLUMN prefix TO filename'); + $this->addSql('ALTER TABLE chill_doc.stored_object ADD type TEXT NOT NULL DEFAULT \'\''); + $this->addSql('ALTER TABLE chill_doc.stored_object ADD key JSON NOT NULL DEFAULT \'{}\''); + $this->addSql('ALTER TABLE chill_doc.stored_object ADD iv JSON NOT NULL DEFAULT \'[]\''); + $this->addSql('ALTER TABLE chill_doc.stored_object ALTER title DROP DEFAULT'); + $this->addSql( + <<<'SQL' + UPDATE chill_doc.stored_object SET filename=sov.filename, type=sov.type, iv=sov.iv, key=sov.key + FROM chill_doc.stored_object_version sov WHERE sov.stored_object_id = stored_object.id + AND sov.version = (SELECT MAX(version) FROM chill_doc.stored_object_version AS sub_sov WHERE sub_sov.stored_object_id = stored_object.id) + SQL + ); + $this->addSql('ALTER TABLE chill_doc.stored_object_version DROP CONSTRAINT FK_C1D55302232D562B'); + $this->addSql('ALTER TABLE chill_doc.stored_object_version DROP CONSTRAINT FK_C1D553023174800F'); + $this->addSql('DROP TABLE chill_doc.stored_object_version'); + } +} From 4fbb7811ac09b63aeeb17e885c3e4f3cf78e0a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Tue, 9 Jul 2024 22:24:33 +0200 Subject: [PATCH 02/25] Refactor StoredObjectDataMapper to use registerVersion method Removed saveHistory and set methods in StoredObjectDataMapper and replaced them with one call to registerVersion. --- .../Form/DataMapper/StoredObjectDataMapper.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Bundle/ChillDocStoreBundle/Form/DataMapper/StoredObjectDataMapper.php b/src/Bundle/ChillDocStoreBundle/Form/DataMapper/StoredObjectDataMapper.php index 170b1ab0a..37d28c7a4 100644 --- a/src/Bundle/ChillDocStoreBundle/Form/DataMapper/StoredObjectDataMapper.php +++ b/src/Bundle/ChillDocStoreBundle/Form/DataMapper/StoredObjectDataMapper.php @@ -57,15 +57,14 @@ class StoredObjectDataMapper implements DataMapperInterface /** @var StoredObject $viewData */ if ($viewData->getFilename() !== $forms['stored_object']->getData()['filename']) { - // we want to keep the previous history - $viewData->saveHistory(); + $viewData->registerVersion( + $forms['stored_object']->getData()['iv'], + $forms['stored_object']->getData()['keyInfos'], + $forms['stored_object']->getData()['type'], + $forms['stored_object']->getData()['filename'], + ); } - $viewData->setFilename($forms['stored_object']->getData()['filename']); - $viewData->setIv($forms['stored_object']->getData()['iv']); - $viewData->setKeyInfos($forms['stored_object']->getData()['keyInfos']); - $viewData->setType($forms['stored_object']->getData()['type']); - if (array_key_exists('title', $forms)) { $viewData->setTitle($forms['title']->getData()); } From 3978ea9a47e0827e860d60c53e2406b7e84220cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Tue, 9 Jul 2024 22:24:55 +0200 Subject: [PATCH 03/25] Update StoredObjectManager to handle versioned StoredObjects The StoredObjectManager and related test cases have been updated to handle versioned StoredObjects, allowing the same methods to work with either a StoredObject or its versions. The changes also involve return information for the write method and enhancements to the write test procedure. This provides more functionality and flexibility for handling StoredObjects in different versions. --- .../Service/StoredObjectManager.php | 98 +++++++----- .../Service/StoredObjectManagerInterface.php | 34 ++++- .../Tests/Service/StoredObjectManagerTest.php | 144 +++++++++++++----- 3 files changed, 199 insertions(+), 77 deletions(-) diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php index 7100ca821..b10f67619 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php @@ -14,6 +14,7 @@ namespace Chill\DocStoreBundle\Service; use Base64Url\Base64Url; use Chill\DocStoreBundle\AsyncUpload\TempUrlGeneratorInterface; use Chill\DocStoreBundle\Entity\StoredObject; +use Chill\DocStoreBundle\Entity\StoredObjectVersion; use Chill\DocStoreBundle\Exception\StoredObjectManagerException; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; @@ -32,10 +33,12 @@ final class StoredObjectManager implements StoredObjectManagerInterface private readonly TempUrlGeneratorInterface $tempUrlGenerator ) {} - public function getLastModified(StoredObject $document): \DateTimeInterface + public function getLastModified(StoredObject|StoredObjectVersion $document): \DateTimeInterface { - if ($this->hasCache($document)) { - $response = $this->getResponseFromCache($document); + $version = $document instanceof StoredObject ? $document->getCurrentVersion() : $document; + + if ($this->hasCache($version)) { + $response = $this->getResponseFromCache($version); } else { try { $response = $this @@ -46,7 +49,7 @@ final class StoredObjectManager implements StoredObjectManagerInterface ->tempUrlGenerator ->generate( Request::METHOD_HEAD, - $document->getFilename() + $version->getFilename() ) ->url ); @@ -58,11 +61,13 @@ final class StoredObjectManager implements StoredObjectManagerInterface return $this->extractLastModifiedFromResponse($response); } - public function getContentLength(StoredObject $document): int + public function getContentLength(StoredObject|StoredObjectVersion $document): int { - if ([] === $document->getKeyInfos()) { - if ($this->hasCache($document)) { - $response = $this->getResponseFromCache($document); + $version = $document instanceof StoredObject ? $document->getCurrentVersion() : $document; + + if (!$this->isVersionEncrypted($version)) { + if ($this->hasCache($version)) { + $response = $this->getResponseFromCache($version); } else { try { $response = $this @@ -73,7 +78,7 @@ final class StoredObjectManager implements StoredObjectManagerInterface ->tempUrlGenerator ->generate( Request::METHOD_HEAD, - $document->getFilename() + $version->getFilename() ) ->url ); @@ -88,10 +93,12 @@ final class StoredObjectManager implements StoredObjectManagerInterface return strlen($this->read($document)); } - public function etag(StoredObject $document): string + public function etag(StoredObject|StoredObjectVersion $document): string { - if ($this->hasCache($document)) { - $response = $this->getResponseFromCache($document); + $version = $document instanceof StoredObject ? $document->getCurrentVersion() : $document; + + if ($this->hasCache($version)) { + $response = $this->getResponseFromCache($version); } else { try { $response = $this @@ -111,12 +118,14 @@ final class StoredObjectManager implements StoredObjectManagerInterface } } - return $this->extractEtagFromResponse($response, $document); + return $this->extractEtagFromResponse($response); } - public function read(StoredObject $document): string + public function read(StoredObject|StoredObjectVersion $document, ?int $version = null): string { - $response = $this->getResponseFromCache($document); + $version = $document instanceof StoredObject ? $document->getCurrentVersion() : $document; + + $response = $this->getResponseFromCache($version); try { $data = $response->getContent(); @@ -124,7 +133,7 @@ final class StoredObjectManager implements StoredObjectManagerInterface throw StoredObjectManagerException::unableToGetResponseContent($e); } - if (false === $this->hasKeysAndIv($document)) { + if (!$this->isVersionEncrypted($version)) { return $data; } @@ -132,9 +141,9 @@ final class StoredObjectManager implements StoredObjectManagerInterface $data, self::ALGORITHM, // TODO: Why using this library and not use base64_decode() ? - Base64Url::decode($document->getKeyInfos()['k']), + Base64Url::decode($version->getKeyInfos()['k']), \OPENSSL_RAW_DATA, - pack('C*', ...$document->getIv()) + pack('C*', ...$version->getIv()) ); if (false === $clearData) { @@ -144,20 +153,25 @@ final class StoredObjectManager implements StoredObjectManagerInterface return $clearData; } - public function write(StoredObject $document, string $clearContent): void + public function write(StoredObject $document, string $clearContent, ?string $contentType = null): StoredObjectVersion { - if ($this->hasCache($document)) { - unset($this->inMemory[$document->getUuid()->toString()]); - } + $newIv = $document->getIv(); + $newKey = $document->getKeyInfos(); + $newType = $contentType ?? $document->getType(); + $version = $document->registerVersion( + $newIv, + $newKey, + $newType + ); - $encryptedContent = $this->hasKeysAndIv($document) + $encryptedContent = $this->isVersionEncrypted($version) ? openssl_encrypt( $clearContent, self::ALGORITHM, // TODO: Why using this library and not use base64_decode() ? - Base64Url::decode($document->getKeyInfos()['k']), + Base64Url::decode($version->getKeyInfos()['k']), \OPENSSL_RAW_DATA, - pack('C*', ...$document->getIv()) + pack('C*', ...$version->getIv()) ) : $clearContent; @@ -176,7 +190,7 @@ final class StoredObjectManager implements StoredObjectManagerInterface ->tempUrlGenerator ->generate( Request::METHOD_PUT, - $document->getFilename() + $version->getFilename() ) ->url, [ @@ -191,6 +205,8 @@ final class StoredObjectManager implements StoredObjectManagerInterface if (Response::HTTP_CREATED !== $response->getStatusCode()) { throw StoredObjectManagerException::invalidStatusCode($response->getStatusCode()); } + + return $version; } public function clearCache(): void @@ -215,12 +231,19 @@ final class StoredObjectManager implements StoredObjectManagerInterface return $date; } + /** + * Extracts the content length from a ResponseInterface object. + * + * Does work only if the object is not encrypted. + * + * @return int the extracted content length as an integer + */ private function extractContentLengthFromResponse(ResponseInterface $response): int { return (int) ($response->getHeaders()['content-length'] ?? ['0'])[0]; } - private function extractEtagFromResponse(ResponseInterface $response, StoredObject $storedObject): ?string + private function extractEtagFromResponse(ResponseInterface $response): ?string { $etag = ($response->getHeaders()['etag'] ?? [''])[0]; @@ -231,7 +254,7 @@ final class StoredObjectManager implements StoredObjectManagerInterface return $etag; } - private function fillCache(StoredObject $document): void + private function fillCache(StoredObjectVersion $document): void { try { $response = $this @@ -254,25 +277,30 @@ final class StoredObjectManager implements StoredObjectManagerInterface throw StoredObjectManagerException::invalidStatusCode($response->getStatusCode()); } - $this->inMemory[$document->getUuid()->toString()] = $response; + $this->inMemory[$this->buildCacheKey($document)] = $response; } - private function getResponseFromCache(StoredObject $document): ResponseInterface + private function buildCacheKey(StoredObjectVersion $storedObjectVersion): string + { + return $storedObjectVersion->getStoredObject()->getUuid()->toString().$storedObjectVersion->getId(); + } + + private function getResponseFromCache(StoredObjectVersion $document): ResponseInterface { if (!$this->hasCache($document)) { $this->fillCache($document); } - return $this->inMemory[$document->getUuid()->toString()]; + return $this->inMemory[$this->buildCacheKey($document)]; } - private function hasCache(StoredObject $document): bool + private function hasCache(StoredObjectVersion $document): bool { - return \array_key_exists($document->getUuid()->toString(), $this->inMemory); + return \array_key_exists($this->buildCacheKey($document), $this->inMemory); } - private function hasKeysAndIv(StoredObject $storedObject): bool + private function isVersionEncrypted(StoredObjectVersion $storedObjectVersion): bool { - return ([] !== $storedObject->getKeyInfos()) && ([] !== $storedObject->getIv()); + return ([] !== $storedObjectVersion->getKeyInfos()) && ([] !== $storedObjectVersion->getIv()); } } diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php index 19ff974f0..4d2f45c33 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php @@ -12,36 +12,56 @@ declare(strict_types=1); namespace Chill\DocStoreBundle\Service; use Chill\DocStoreBundle\Entity\StoredObject; +use Chill\DocStoreBundle\Entity\StoredObjectVersion; use Chill\DocStoreBundle\Exception\StoredObjectManagerException; interface StoredObjectManagerInterface { - public function getLastModified(StoredObject $document): \DateTimeInterface; + /** + * @param StoredObject|StoredObjectVersion $document if a StoredObject is given, the last version will be used + */ + public function getLastModified(StoredObject|StoredObjectVersion $document): \DateTimeInterface; - public function getContentLength(StoredObject $document): int; + /** + * @param StoredObject|StoredObjectVersion $document if a StoredObject is given, the last version will be used + */ + public function getContentLength(StoredObject|StoredObjectVersion $document): int; /** * Get the content of a StoredObject. * - * @param StoredObject $document the document + * @param StoredObject|StoredObjectVersion $document if a StoredObject is given, the last version will be used * * @return string the retrieved content in clear * * @throws StoredObjectManagerException if unable to read or decrypt the content */ - public function read(StoredObject $document): string; + public function read(StoredObject|StoredObjectVersion $document): string; /** * Set the content of a StoredObject. * * @param StoredObject $document the document - * @param $clearContent The content to store in clear + * @param string $clearContent The content to store in clear + * @param string|null $contentType The new content type. If set to null, the content-type is supposed not to change. If there is no content type, an empty string will be used. + * + * @return StoredObjectVersion the newly created @see{StoredObjectVersion} for the given @see{StoredObject} * * @throws StoredObjectManagerException */ - public function write(StoredObject $document, string $clearContent): void; + public function write(StoredObject $document, string $clearContent, ?string $contentType = null): StoredObjectVersion; - public function etag(StoredObject $document): string; + /** + * return or compute the etag for the document. + * + * @param StoredObject|StoredObjectVersion $document if a StoredObject is given, the last version will be used + * + * @return string the etag of this document + */ + public function etag(StoredObject|StoredObjectVersion $document): string; + /** + * Clears the cache for the stored object. + */ public function clearCache(): void; } diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php index c5cc8185e..c11a06f9b 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php @@ -31,23 +31,25 @@ use Symfony\Contracts\HttpClient\HttpClientInterface; */ final class StoredObjectManagerTest extends TestCase { - public static function getDataProvider(): \Generator + public static function getDataProviderForRead(): \Generator { /* HAPPY SCENARIO */ // Encrypted object yield [ (new StoredObject()) - ->setFilename('encrypted.txt') - ->setKeyInfos(['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')]) - ->setIv(unpack('C*', 'abcdefghijklmnop')), + ->registerVersion( + unpack('C*', 'abcdefghijklmnop'), + ['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')], + filename: 'encrypted.txt' + )->getStoredObject(), hex2bin('741237d255fd4f7eddaaa9058912a84caae28a41b10b34d4e3e3abe41d3b9b47cb0dd8f22c3c883d4f0e9defa75ff662'), // Binary encoded string 'The quick brown fox jumps over the lazy dog', // clear ]; // Non-encrypted object yield [ - (new StoredObject())->setFilename('non-encrypted.txt'), // The StoredObject + (new StoredObject())->registerVersion(filename: 'non-encrypted.txt')->getStoredObject(), // The StoredObject 'The quick brown fox jumps over the lazy dog', // Encrypted 'The quick brown fox jumps over the lazy dog', // Clear ]; @@ -57,9 +59,11 @@ final class StoredObjectManagerTest extends TestCase // Encrypted object with issue during HTTP communication yield [ (new StoredObject()) - ->setFilename('error_during_http_request.txt') - ->setKeyInfos(['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')]) - ->setIv(unpack('C*', 'abcdefghijklmnop')), + ->registerVersion( + unpack('C*', 'abcdefghijklmnop'), + ['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')], + filename: 'error_during_http_request.txt' + )->getStoredObject(), hex2bin('741237d255fd4f7eddaaa9058912a84caae28a41b10b34d4e3e3abe41d3b9b47cb0dd8f22c3c883d4f0e9defa75ff662'), // Binary encoded string 'The quick brown fox jumps over the lazy dog', // clear StoredObjectManagerException::class, @@ -68,9 +72,11 @@ final class StoredObjectManagerTest extends TestCase // Encrypted object with issue during HTTP communication: Invalid status code yield [ (new StoredObject()) - ->setFilename('invalid_statuscode.txt') - ->setKeyInfos(['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')]) - ->setIv(unpack('C*', 'abcdefghijklmnop')), + ->registerVersion( + unpack('C*', 'abcdefghijklmnop'), + ['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')], + filename: 'invalid_statuscode.txt' + )->getStoredObject(), hex2bin('741237d255fd4f7eddaaa9058912a84caae28a41b10b34d4e3e3abe41d3b9b47cb0dd8f22c3c883d4f0e9defa75ff662'), // Binary encoded string 'The quick brown fox jumps over the lazy dog', // clear StoredObjectManagerException::class, @@ -79,17 +85,73 @@ final class StoredObjectManagerTest extends TestCase // Erroneous encrypted: Unable to decrypt exception. yield [ (new StoredObject()) - ->setFilename('unable_to_decrypt.txt') - ->setKeyInfos(['k' => base64_encode('WRONG_PASS_PHRASE')]) - ->setIv(unpack('C*', 'abcdefghijklmnop')), + ->registerVersion( + unpack('C*', 'abcdefghijklmnop'), + ['k' => base64_encode('WRONG_PASS_PHRASE')], + filename: 'unable_to_decrypt.txt' + )->getStoredObject(), 'WRONG_ENCODED_VALUE', // Binary encoded string 'The quick brown fox jumps over the lazy dog', // clear StoredObjectManagerException::class, ]; } + public static function getDataProviderForWrite(): \Generator + { + /* HAPPY SCENARIO */ + + // Encrypted object + yield [ + (new StoredObject()) + ->registerVersion( + unpack('C*', 'abcdefghijklmnop'), + ['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')], + filename: 'encrypted.txt' + )->getStoredObject(), + hex2bin('741237d255fd4f7eddaaa9058912a84caae28a41b10b34d4e3e3abe41d3b9b47cb0dd8f22c3c883d4f0e9defa75ff662'), // Binary encoded string + 'The quick brown fox jumps over the lazy dog', // clear + ]; + + // Non-encrypted object + yield [ + (new StoredObject())->registerVersion(filename: 'non-encrypted.txt')->getStoredObject(), // The StoredObject + 'The quick brown fox jumps over the lazy dog', // Encrypted + 'The quick brown fox jumps over the lazy dog', // Clear + ]; + + /* UNHAPPY SCENARIO */ + + // Encrypted object with issue during HTTP communication + yield [ + (new StoredObject()) + ->registerVersion( + unpack('C*', 'abcdefghijklmnop'), + ['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')], + filename: 'error_during_http_request.txt' + )->getStoredObject(), + hex2bin('741237d255fd4f7eddaaa9058912a84caae28a41b10b34d4e3e3abe41d3b9b47cb0dd8f22c3c883d4f0e9defa75ff662'), // Binary encoded string + 'The quick brown fox jumps over the lazy dog', // clear + StoredObjectManagerException::class, + -1, + ]; + + // Encrypted object with issue during HTTP communication: Invalid status code + yield [ + (new StoredObject()) + ->registerVersion( + unpack('C*', 'abcdefghijklmnop'), + ['k' => base64_encode('S9NIHMaFHOWzLPez3jZOIHBaNfBrMQUR5zvqBz6kme8')], + filename: 'invalid_statuscode.txt' + )->getStoredObject(), + hex2bin('741237d255fd4f7eddaaa9058912a84caae28a41b10b34d4e3e3abe41d3b9b47cb0dd8f22c3c883d4f0e9defa75ff662'), // Binary encoded string + 'The quick brown fox jumps over the lazy dog', // clear + StoredObjectManagerException::class, + 408, + ]; + } + /** - * @dataProvider getDataProvider + * @dataProvider getDataProviderForRead */ public function testRead(StoredObject $storedObject, string $encodedContent, string $clearContent, ?string $exceptionClass = null) { @@ -103,19 +165,37 @@ final class StoredObjectManagerTest extends TestCase } /** - * @dataProvider getDataProvider + * @dataProvider getDataProviderForWrite */ - public function testWrite(StoredObject $storedObject, string $encodedContent, string $clearContent, ?string $exceptionClass = null) + public function testWrite(StoredObject $storedObject, string $encodedContent, string $clearContent, ?string $exceptionClass = null, ?int $errorCode = null) { if (null !== $exceptionClass) { $this->expectException($exceptionClass); } - $storedObjectManager = $this->getSubject($storedObject, $encodedContent); + $previousVersion = $storedObject->getCurrentVersion(); + $previousFilename = $previousVersion->getFilename(); - $storedObjectManager->write($storedObject, $clearContent); + $client = new MockHttpClient(function ($method, $url, $options) use ($encodedContent, $previousFilename, $errorCode) { + self::assertEquals('PUT', $method); + self::assertStringStartsWith('https://example.com/', $url); + self::assertStringNotContainsString($previousFilename, $url, 'test that the PUT operation is not performed on the same file'); + self::assertArrayHasKey('body', $options); + self::assertEquals($encodedContent, $options['body']); - self::assertEquals($clearContent, $storedObjectManager->read($storedObject)); + if (-1 === $errorCode) { + throw new TransportException(); + } + + return new MockResponse('', ['http_code' => $errorCode ?? 201]); + }); + + $storedObjectManager = new StoredObjectManager($client, $this->getTempUrlGenerator($storedObject)); + + $newVersion = $storedObjectManager->write($storedObject, $clearContent); + + self::assertNotSame($previousVersion, $newVersion); + self::assertSame($storedObject->getCurrentVersion(), $newVersion); } public function testWriteWithDeleteAt() @@ -170,19 +250,6 @@ final class StoredObjectManagerTest extends TestCase } } - if (Request::METHOD_PUT === $method) { - switch ($url) { - case 'https://example.com/non-encrypted.txt': - case 'https://example.com/encrypted.txt': - return new MockResponse($encodedContent, ['http_code' => 201]); - - case 'https://example.com/error_during_http_request.txt': - throw new TransportException('error_during_http_request.txt'); - case 'https://example.com/invalid_statuscode.txt': - return new MockResponse($encodedContent, ['http_code' => 404]); - } - } - return new MockResponse('Not found'); }; @@ -209,9 +276,16 @@ final class StoredObjectManagerTest extends TestCase $tempUrlGenerator = $this->createMock(TempUrlGeneratorInterface::class); $tempUrlGenerator + ->expects($this->atLeastOnce()) ->method('generate') - ->withAnyParameters() - ->willReturn($response); + ->with($this->logicalOr($this->identicalTo('GET'), $this->identicalTo('PUT')), $this->isType('string')) + ->willReturnCallback(function (string $method, string $objectName) { + return new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours') + ); + }); return $tempUrlGenerator; } From e21db73b84b812c08c4ca2ec02a17862a71c2e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Tue, 9 Jul 2024 22:25:52 +0200 Subject: [PATCH 04/25] Refactor and simplify document management functionality to adapt to StoredObject versioning This commit includes several updates to the document management functionality within ChillWopiBundle and ChillDocGeneratorBundle, refactoring for simplicity and improved readability. --- .../DataFixtures/ORM/LoadDocGeneratorTemplate.php | 13 ++++++++----- .../Service/Generator/Generator.php | 8 ++------ .../src/Service/Wopi/ChillDocumentManager.php | 10 +++++----- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/Bundle/ChillDocGeneratorBundle/DataFixtures/ORM/LoadDocGeneratorTemplate.php b/src/Bundle/ChillDocGeneratorBundle/DataFixtures/ORM/LoadDocGeneratorTemplate.php index 97d436c5e..f38102a10 100644 --- a/src/Bundle/ChillDocGeneratorBundle/DataFixtures/ORM/LoadDocGeneratorTemplate.php +++ b/src/Bundle/ChillDocGeneratorBundle/DataFixtures/ORM/LoadDocGeneratorTemplate.php @@ -54,12 +54,15 @@ class LoadDocGeneratorTemplate extends AbstractFixture ]; foreach ($templates as $template) { - $newStoredObj = (new StoredObject()) - ->setFilename($template['file']['filename']) - ->setKeyInfos(json_decode($template['file']['key'], true)) - ->setIv(json_decode($template['file']['iv'], true)) + $newStoredObj = (new StoredObject()); + + $newStoredObj ->setCreatedAt(new \DateTime('today')) - ->setType($template['file']['type']); + ->registerVersion( + json_decode($template['file']['key'], true), + json_decode($template['file']['iv'], true), + $template['file']['type'], + ); $manager->persist($newStoredObj); diff --git a/src/Bundle/ChillDocGeneratorBundle/Service/Generator/Generator.php b/src/Bundle/ChillDocGeneratorBundle/Service/Generator/Generator.php index 127d135bb..d4ca5cf2e 100644 --- a/src/Bundle/ChillDocGeneratorBundle/Service/Generator/Generator.php +++ b/src/Bundle/ChillDocGeneratorBundle/Service/Generator/Generator.php @@ -134,13 +134,11 @@ class Generator implements GeneratorInterface $content = Yaml::dump($data, 6); /* @var StoredObject $destinationStoredObject */ $destinationStoredObject - ->setType('application/yaml') - ->setFilename(sprintf('%s_yaml', uniqid('doc_', true))) ->setStatus(StoredObject::STATUS_READY) ; try { - $this->storedObjectManager->write($destinationStoredObject, $content); + $this->storedObjectManager->write($destinationStoredObject, $content, 'application/yaml'); } catch (StoredObjectManagerException $e) { $destinationStoredObject->addGenerationErrors($e->getMessage()); @@ -174,13 +172,11 @@ class Generator implements GeneratorInterface /* @var StoredObject $destinationStoredObject */ $destinationStoredObject - ->setType($template->getFile()->getType()) - ->setFilename(sprintf('%s_odt', uniqid('doc_', true))) ->setStatus(StoredObject::STATUS_READY) ; try { - $this->storedObjectManager->write($destinationStoredObject, $generatedResource); + $this->storedObjectManager->write($destinationStoredObject, $generatedResource, $template->getFile()->getType()); } catch (StoredObjectManagerException $e) { $destinationStoredObject->addGenerationErrors($e->getMessage()); diff --git a/src/Bundle/ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php b/src/Bundle/ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php index 2b569014d..574fee75b 100644 --- a/src/Bundle/ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php +++ b/src/Bundle/ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php @@ -50,17 +50,15 @@ final readonly class ChillDocumentManager implements DocumentManagerInterface // Mime types / extension handling. $mimeTypes = new MimeTypes(); - $mimeTypes->getMimeTypes($data['extension']); - $document->setType(reset($mimeTypes)); - - $document->setFilename($data['name']); + $types = $mimeTypes->getMimeTypes($data['extension']); + $mimeType = array_values($types)[0] ?? ''; $this->entityManager->persist($document); $this->entityManager->flush(); // TODO : Ask proper mapping. // Available: basename, name, extension, content, size - $this->setContent($document, $data['content']); + $this->storedObjectManager->write($document, $data['content'], $mimeType); return $document; } @@ -194,5 +192,7 @@ final readonly class ChillDocumentManager implements DocumentManagerInterface private function setContent(StoredObject $storedObject, string $content): void { $this->storedObjectManager->write($storedObject, $content); + + $this->entityManager->flush(); } } From 5fefe09a39805da7b9e54b29f4ce407aa1790371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Thu, 11 Jul 2024 10:43:20 +0200 Subject: [PATCH 05/25] Fix test --- .../Tests/AsyncUpload/Templating/AsyncUploadExtensionTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Templating/AsyncUploadExtensionTest.php b/src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Templating/AsyncUploadExtensionTest.php index d39809a12..77c5f72ec 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Templating/AsyncUploadExtensionTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Templating/AsyncUploadExtensionTest.php @@ -70,7 +70,7 @@ class AsyncUploadExtensionTest extends KernelTestCase public static function dataProviderStoredObject(): iterable { - yield [(new StoredObject())->setFilename('blabla')]; + yield [(new StoredObject())->registerVersion(filename: 'blabla')->getStoredObject()]; yield ['blabla']; } From ce5659219a6d528f4e69eb6cf6da4a5f062a9c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Thu, 11 Jul 2024 11:04:04 +0200 Subject: [PATCH 06/25] Fix test --- .../Tests/Controller/WebdavControllerTest.php | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Controller/WebdavControllerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Controller/WebdavControllerTest.php index d116add92..5d6565108 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Controller/WebdavControllerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Controller/WebdavControllerTest.php @@ -14,6 +14,7 @@ namespace Chill\DocStoreBundle\Tests\Controller; use Chill\DocStoreBundle\Controller\WebdavController; use Chill\DocStoreBundle\Entity\StoredObject; use Chill\DocStoreBundle\Service\StoredObjectManagerInterface; +use Doctrine\ORM\EntityManagerInterface; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; use Ramsey\Uuid\Uuid; @@ -39,20 +40,25 @@ class WebdavControllerTest extends KernelTestCase $this->engine = self::getContainer()->get(\Twig\Environment::class); } - private function buildController(): WebdavController + private function buildController(EntityManagerInterface $entityManager = null): WebdavController { $storedObjectManager = new MockedStoredObjectManager(); $security = $this->prophesize(Security::class); $security->isGranted(Argument::in(['EDIT', 'SEE']), Argument::type(StoredObject::class)) ->willReturn(true); - return new WebdavController($this->engine, $storedObjectManager, $security->reveal()); + if (null === $entityManager) { + $entityManager = $this->createMock(EntityManagerInterface::class); + } + + return new WebdavController($this->engine, $storedObjectManager, $security->reveal(), $entityManager); } private function buildDocument(): StoredObject { $object = (new StoredObject()) - ->setType('application/vnd.oasis.opendocument.text'); + ->registerVersion(type: 'application/vnd.oasis.opendocument.text') + ->getStoredObject(); $reflectionObject = new \ReflectionClass($object); $reflectionObjectUuid = $reflectionObject->getProperty('uuid'); @@ -384,24 +390,27 @@ class WebdavControllerTest extends KernelTestCase class MockedStoredObjectManager implements StoredObjectManagerInterface { - public function getLastModified(StoredObject $document): \DateTimeInterface + public function getLastModified(StoredObject|\Chill\DocStoreBundle\Entity\StoredObjectVersion $document): \DateTimeInterface { - return new \DateTimeImmutable('2023-09-13T14:15'); + return new \DateTimeImmutable('2023-09-13T14:15', new \DateTimeZone('+02:00')); } - public function getContentLength(StoredObject $document): int + public function getContentLength(StoredObject|\Chill\DocStoreBundle\Entity\StoredObjectVersion $document): int { return 5; } - public function read(StoredObject $document): string + public function read(StoredObject|\Chill\DocStoreBundle\Entity\StoredObjectVersion $document): string { return 'abcde'; } - public function write(StoredObject $document, string $clearContent): void {} + public function write(StoredObject $document, string $clearContent, ?string $contentType = null): \Chill\DocStoreBundle\Entity\StoredObjectVersion + { + return $document->registerVersion(); + } - public function etag(StoredObject $document): string + public function etag(StoredObject|\Chill\DocStoreBundle\Entity\StoredObjectVersion $document): string { return 'ab56b4d92b40713acc5af89985d4b786'; } From 1b16d4fe3bc8da05f68184a9bd58f39d7f657379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Thu, 11 Jul 2024 14:48:53 +0200 Subject: [PATCH 07/25] Flush entities when storing document using webdav / put operation The WebdavController has been updated to flush the EntityManager after writing a document, while its tests have been adjusted correspondingly. A new test for the document PUT operation has also been added, which ensures the EntityManager flushes and the StoredObjectManager writes to this document. --- .../Controller/WebdavController.php | 4 ++ .../Tests/Controller/WebdavControllerTest.php | 37 ++++++++++++++++--- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/Bundle/ChillDocStoreBundle/Controller/WebdavController.php b/src/Bundle/ChillDocStoreBundle/Controller/WebdavController.php index 6e105f9ee..022d544cb 100644 --- a/src/Bundle/ChillDocStoreBundle/Controller/WebdavController.php +++ b/src/Bundle/ChillDocStoreBundle/Controller/WebdavController.php @@ -16,6 +16,7 @@ use Chill\DocStoreBundle\Dav\Response\DavResponse; use Chill\DocStoreBundle\Entity\StoredObject; use Chill\DocStoreBundle\Security\Authorization\StoredObjectRoleEnum; use Chill\DocStoreBundle\Service\StoredObjectManagerInterface; +use Doctrine\ORM\EntityManagerInterface; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException; @@ -42,6 +43,7 @@ final readonly class WebdavController private \Twig\Environment $engine, private StoredObjectManagerInterface $storedObjectManager, private Security $security, + private EntityManagerInterface $entityManager, ) { $this->requestAnalyzer = new PropfindRequestAnalyzer(); } @@ -201,6 +203,8 @@ final readonly class WebdavController $this->storedObjectManager->write($storedObject, $request->getContent()); + $this->entityManager->flush(); + return new DavResponse('', Response::HTTP_NO_CONTENT); } diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Controller/WebdavControllerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Controller/WebdavControllerTest.php index 5d6565108..cf1d8c814 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Controller/WebdavControllerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Controller/WebdavControllerTest.php @@ -40,17 +40,20 @@ class WebdavControllerTest extends KernelTestCase $this->engine = self::getContainer()->get(\Twig\Environment::class); } - private function buildController(EntityManagerInterface $entityManager = null): WebdavController + private function buildController(?EntityManagerInterface $entityManager = null, ?StoredObjectManagerInterface $storedObjectManager = null): WebdavController { - $storedObjectManager = new MockedStoredObjectManager(); - $security = $this->prophesize(Security::class); - $security->isGranted(Argument::in(['EDIT', 'SEE']), Argument::type(StoredObject::class)) - ->willReturn(true); + if (null === $storedObjectManager) { + $storedObjectManager = new MockedStoredObjectManager(); + } if (null === $entityManager) { $entityManager = $this->createMock(EntityManagerInterface::class); } + $security = $this->prophesize(Security::class); + $security->isGranted(Argument::in(['SEE_AND_EDIT', 'SEE']), Argument::type(StoredObject::class)) + ->willReturn(true); + return new WebdavController($this->engine, $storedObjectManager, $security->reveal(), $entityManager); } @@ -165,6 +168,30 @@ class WebdavControllerTest extends KernelTestCase self::assertEquals(5, $response->headers->get('content-length')); } + public function testPutDocument(): void + { + $document = $this->buildDocument(); + $entityManager = $this->createMock(EntityManagerInterface::class); + $storedObjectManager = $this->createMock(StoredObjectManagerInterface::class); + + // entity manager must be flushed + $entityManager->expects($this->once()) + ->method('flush'); + + // object must be written by StoredObjectManager + $storedObjectManager->expects($this->once()) + ->method('write') + ->with($this->identicalTo($document), $this->identicalTo('1234')); + + $controller = $this->buildController($entityManager, $storedObjectManager); + + $request = new Request(content: '1234'); + $response = $controller->putDocument($document, $request); + + self::assertEquals(204, $response->getStatusCode()); + self::assertEquals('', $response->getContent()); + } + public static function generateDataPropfindDocument(): iterable { $content = From 2feea24c41efa07879aedd821c2b81a59acad4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Fri, 12 Jul 2024 00:00:13 +0200 Subject: [PATCH 08/25] Add Collabora configuration guide and NullProofValidator class A new document detailing the configuration steps for enabling Collabora in development has been added. This includes setting up ngrok and configuring both the Collabora and Symfony servers. Furthermore, a new class, NullProofValidator, has been created for validation during the development process. This class always returns true, making it useful for debugging purposes. --- .../installation/enable-collabora-for-dev.rst | 125 ++++++++++++++++++ .../src/Service/Wopi/NullProofValidator.php | 28 ++++ 2 files changed, 153 insertions(+) create mode 100644 docs/source/installation/enable-collabora-for-dev.rst create mode 100644 src/Bundle/ChillWopiBundle/src/Service/Wopi/NullProofValidator.php diff --git a/docs/source/installation/enable-collabora-for-dev.rst b/docs/source/installation/enable-collabora-for-dev.rst new file mode 100644 index 000000000..17a9ae1cc --- /dev/null +++ b/docs/source/installation/enable-collabora-for-dev.rst @@ -0,0 +1,125 @@ + +Enable CODE for development +=========================== + +For editing a document, there must be a way to communicate between the collabora server and the symfony server, in +both direction. The domain name should also be the same for collabora server and for the browser which access to the +online editor. + +Using ngrok (or other http tunnel) +---------------------------------- + +One can configure a tunnel server to expose your local install to the web, and access to your local server using the +tunnel url. + +Start ngrok +^^^^^^^^^^^ + +This can be achieve using `ngrok `_. + +.. note:: + + The configuration of ngrok is outside of the scope of this document. Refers to the ngrok's documentation. + +.. code-block:: bash + + # ensuring that your server is running through http and port 8000 + ngrok http 8000 + # then open the link given by the ngrok utility and you should reach your app + +At this step, ensure that you can reach your local app using the ngrok url. + +Configure Collabora +^^^^^^^^^^^^^^^^^^^ + +The collabora server must be executed online and configure to access to your ngrok installation. Ensure that the aliasgroup +exists for your ngrok application (`See the CODE documentation: `_). + +Configure your app +^^^^^^^^^^^^^^^^^^ + +Set the :code:`EDITOR_SERVER` variable to point to your collabora server, this should be done in your :code:`.env.local` file. + +At this point, everything must be fine. In case of errors, watch the log from your collabora server, use the `profiler `_ +to debug the requests. + +.. note:: + + In case of error while validating proof (you'll see those message in the collabora's logs), you can temporarily disable + the proof validation adding this code snippet in `config/services.yaml`: + + .. code-block:: yaml + + when@dev: + # add only in dev environment, to avoid security problems + services: + ChampsLibres\WopiLib\Contract\Service\ProofValidatorInterface: + # this class will always validate proof + alias: Chill\WopiBundle\Service\Wopi\NullProofValidator + +With a local CODE image +----------------------- + +.. warning:: + + This configuration is not sure, and must be refined. The documentation does not seems to be entirely valid. + +Use a local domain name and https for your app +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Use the proxy feature from embedded symfony server to run your app. `See the dedicated doc ` + +Configure also the `https certificate `_ + +In this example, your local domain name will be :code:`my-domain` and the url will be :code:`https://my-domain.wip`. + +Ensure that the proxy is running. + +Create a certificate database for collabora +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Collabora must validate your certificate generated by symfony console. For that, you need `to create a NSS database ` +and configure collabora to use it. + +At first, export the certificate for symfony development. Use the graphical interface from your browser to get the +certificate as a PEM file. + +.. code-block:: bash + + # create your database in a custom directory + mkdir /path/to/your/directory + certutil -N -d /path/to/your/directory + cat /path/to/your/ca.crt | certutil -d . -A symfony -t -t C,P,C,u,w -a + +Launch CODE properly configured + +.. code-block:: yaml + + collabora: + image: collabora/code:latest + environment: + - SLEEPFORDEBUGGER=0 + - DONT_GEN_SSL_CERT="True" + # add path to the database + - extra_params=--o:ssl.enable=false --o:ssl.termination=false --o:logging.level=7 -o:certificates.database_path=/etc/custom-certificates/nss-database + - username=admin + - password=admin + - dictionaries=en_US + - aliasgroup1=https://my-domain.wip + ports: + - "127.0.0.1:9980:9980" + volumes: + - "/path/to/your/directory/nss-database:/etc/custom-certificates/nss-database" + extra_hosts: + - "my-domain.wip:host-gateway" + +Configure your app +^^^^^^^^^^^^^^^^^^ + +Into your :code:`.env.local` file: + +.. code-block:: env + + EDITOR_SERVER=http://${COLLABORA_HOST}:${COLLABORA_PORT} + +At this step, you should be able to edit a document through collabora. diff --git a/src/Bundle/ChillWopiBundle/src/Service/Wopi/NullProofValidator.php b/src/Bundle/ChillWopiBundle/src/Service/Wopi/NullProofValidator.php new file mode 100644 index 000000000..3fc294728 --- /dev/null +++ b/src/Bundle/ChillWopiBundle/src/Service/Wopi/NullProofValidator.php @@ -0,0 +1,28 @@ + Date: Fri, 12 Jul 2024 00:00:44 +0200 Subject: [PATCH 09/25] Update random_bytes length in filename and prefix generation The size of the random byte string used in the generateFilename method of StoredObjectVersion has been reduced from 16 to 8. Conversely, the size of the random byte string used in the generatePrefix method of StoredObject has been increased from 8 to 32. The naming generation fit better with the usage, as 16bytes are generated for each file (more version), and less for the version. --- src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php | 2 +- src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php | 2 +- .../ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php index f37829a26..4fec9f6ca 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php @@ -354,7 +354,7 @@ class StoredObject implements Document, TrackCreationInterface public static function generatePrefix(): string { try { - return base_convert(bin2hex(random_bytes(8)), 16, 36); + return base_convert(bin2hex(random_bytes(32)), 16, 36); } catch (RandomException $e) { return uniqid(more_entropy: true); } diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php index c280b30a3..2b8852513 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php @@ -82,7 +82,7 @@ class StoredObjectVersion implements TrackCreationInterface public static function generateFilename(StoredObjectVersion $storedObjectVersion): string { try { - $suffix = base_convert(bin2hex(random_bytes(16)), 16, 36); + $suffix = base_convert(bin2hex(random_bytes(8)), 16, 36); } catch (RandomException $e) { $suffix = uniqid(more_entropy: true); } diff --git a/src/Bundle/ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php b/src/Bundle/ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php index 574fee75b..639133976 100644 --- a/src/Bundle/ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php +++ b/src/Bundle/ChillWopiBundle/src/Service/Wopi/ChillDocumentManager.php @@ -104,7 +104,7 @@ final readonly class ChillDocumentManager implements DocumentManagerInterface throw new \Error('Unknown mimetype for stored document.'); } - return sprintf('%s.%s', $document->getFilename(), reset($exts)); + return sprintf('%s.%s', $document->getPrefix(), reset($exts)); } /** From 67d24cb951375fa255226f44f5251453de476c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Fri, 12 Jul 2024 00:26:40 +0200 Subject: [PATCH 10/25] Use "createdAt" from database to compute the last modified datetime in StoredObjectManager The code has been updated to use 'createdAt' from StoredObjectVersion entity in StoredObjectManager. Specifically, if a 'createdAt' datetime is set, we return that datetime. This change also includes corresponding test cases to validate the functionality. The situation helps deal with files created before July 2024. --- .../Service/StoredObjectManager.php | 8 +++ .../Tests/Service/StoredObjectManagerTest.php | 61 ++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php index b10f67619..2266bb0e2 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php @@ -37,6 +37,14 @@ final class StoredObjectManager implements StoredObjectManagerInterface { $version = $document instanceof StoredObject ? $document->getCurrentVersion() : $document; + if (null !== $createdAt = $version->getCreatedAt()) { + // as a createdAt datetime is set, return the date and time from database + return $createdAt; + } + + // if no createdAt version exists in the database, we fetch the date and time from the + // file. This situation happens for files created before July 2024. + if ($this->hasCache($version)) { $response = $this->getResponseFromCache($version); } else { diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php index c11a06f9b..3725f910f 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php @@ -233,6 +233,64 @@ final class StoredObjectManagerTest extends TestCase $manager->write($storedObject, 'ok'); } + public function testGetLastModifiedWithDateTimeInEntity(): void + { + $version = ($storedObject = new StoredObject())->registerVersion(); + $version->setCreatedAt(new \DateTimeImmutable('2024-07-09 15:09:47', new \DateTimeZone('+00:00'))); + $client = $this->createMock(HttpClientInterface::class); + $client->expects(self::never())->method('request')->withAnyParameters(); + + $tempUrlGenerator = $this->createMock(TempUrlGeneratorInterface::class); + $tempUrlGenerator + ->expects($this->never()) + ->method('generate') + ->with($this->identicalTo('HEAD'), $this->isType('string')) + ->willReturnCallback(function (string $method, string $objectName) { + return new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours'), + $objectName + ); + }); + + $manager = new StoredObjectManager($client, $tempUrlGenerator); + + $actual = $manager->getLastModified($storedObject); + + self::assertEquals(new \DateTimeImmutable('2024-07-09 15:09:47 GMT'), $actual); + } + + public function testGetLastModifiedWithDateTimeFromResponse(): void + { + $storedObject = (new StoredObject())->registerVersion()->getStoredObject(); + + $client = new MockHttpClient( + new MockResponse('', ['http_code' => 200, 'response_headers' => [ + 'last-modified' => 'Tue, 09 Jul 2024 15:09:47 GMT', + ]]) + ); + + $tempUrlGenerator = $this->createMock(TempUrlGeneratorInterface::class); + $tempUrlGenerator + ->expects($this->atLeastOnce()) + ->method('generate') + ->with($this->identicalTo('HEAD'), $this->isType('string')) + ->willReturnCallback(function (string $method, string $objectName) { + return new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours') + ); + }); + + $manager = new StoredObjectManager($client, $tempUrlGenerator); + + $actual = $manager->getLastModified($storedObject); + + self::assertEquals(new \DateTimeImmutable('2024-07-09 15:09:47 GMT'), $actual); + } + private function getHttpClient(string $encodedContent): HttpClientInterface { $callback = static function ($method, $url, $options) use ($encodedContent) { @@ -283,7 +341,8 @@ final class StoredObjectManagerTest extends TestCase return new SignedUrl( $method, 'https://example.com/'.$objectName, - new \DateTimeImmutable('1 hours') + new \DateTimeImmutable('1 hours'), + $objectName ); }); From c38f7c11791542e9a54f3b0869f52b97164f3476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 15 Jul 2024 15:54:26 +0200 Subject: [PATCH 11/25] Add functionality to delete old versions of documents This commit introduces a feature that automatically deletes old versions of StoredObjects in the Chill application. A cron job, "RemoveOldVersionCronJob", has been implemented to delete versions older than 90 days. A message handler, "RemoveOldVersionMessageHandler", has been added to handle deletion requests. Furthermore, unit tests for the new functionality have been provided. --- .../StoredObjectVersionRepository.php | 92 ++++++++++++++++ .../RemoveOldVersionCronJob.php | 60 ++++++++++ .../RemoveOldVersionMessage.php | 19 ++++ .../RemoveOldVersionMessageHandler.php | 54 +++++++++ .../Service/StoredObjectManager.php | 17 +++ .../Service/StoredObjectManagerInterface.php | 5 + .../StoredObjectVersionRepositoryTest.php | 43 ++++++++ .../RemoveOldVersionCronJobTest.php | 104 ++++++++++++++++++ .../RemoveOldVersionMessageHandlerTest.php | 51 +++++++++ .../Tests/Service/StoredObjectManagerTest.php | 31 ++++++ 10 files changed, 476 insertions(+) create mode 100644 src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php create mode 100644 src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionCronJob.php create mode 100644 src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessage.php create mode 100644 src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessageHandler.php create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/Service/Repository/StoredObjectVersionRepositoryTest.php create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php diff --git a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php new file mode 100644 index 000000000..047b024a9 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php @@ -0,0 +1,92 @@ + + */ +class StoredObjectVersionRepository implements ObjectRepository +{ + private EntityRepository $repository; + + private Connection $connection; + + public function __construct(EntityManagerInterface $entityManager) + { + $this->repository = $entityManager->getRepository(StoredObjectVersion::class); + $this->connection = $entityManager->getConnection(); + } + + public function find($id): ?StoredObjectVersion + { + return $this->repository->find($id); + } + + public function findAll(): array + { + return $this->repository->findAll(); + } + + public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): array + { + return $this->repository->findBy($criteria, $orderBy, $limit, $offset); + } + + public function findOneBy(array $criteria): ?StoredObjectVersion + { + return $this->repository->findOneBy($criteria); + } + + /** + * Finds the IDs of versions older than a given date and that are not the last version. + * + * Those version are good candidates for a deletion. + * + * @param \DateTimeImmutable $beforeDate the date to compare versions against + * + * @return iterable returns an iterable with the IDs of the versions + */ + public function findIdsByVersionsOlderThanDateAndNotLastVersion(\DateTimeImmutable $beforeDate): iterable + { + $results = $this->connection->executeQuery( + self::QUERY_FIND_IDS_BY_VERSIONS_OLDER_THAN_DATE_AND_NOT_LAST_VERSION, + [$beforeDate], + [Types::DATETIME_IMMUTABLE] + ); + + foreach ($results->iterateAssociative() as $row) { + yield $row['sov_id']; + } + } + + private const QUERY_FIND_IDS_BY_VERSIONS_OLDER_THAN_DATE_AND_NOT_LAST_VERSION = <<<'SQL' + SELECT + sov.id AS sov_id + FROM chill_doc.stored_object_version sov + WHERE + sov.createdat < ?::timestamp + AND + sov.version < (SELECT MAX(sub_sov.version) FROM chill_doc.stored_object_version sub_sov WHERE sub_sov.stored_object_id = sov.stored_object_id) + SQL; + + public function getClassName(): string + { + return StoredObjectVersion::class; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionCronJob.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionCronJob.php new file mode 100644 index 000000000..d190a4e45 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionCronJob.php @@ -0,0 +1,60 @@ +clock->now() >= $cronJobExecution->getLastEnd()->add(new \DateInterval('P1D')); + } + + public function getKey(): string + { + return self::KEY; + } + + public function run(array $lastExecutionData): ?array + { + $deleteBeforeDate = $this->clock->now()->sub(new \DateInterval(self::KEEP_INTERVAL)); + $maxDeleted = $lastExecutionData[self::LAST_DELETED_KEY] ?? 0; + + foreach ($this->storedObjectVersionRepository->findIdsByVersionsOlderThanDateAndNotLastVersion($deleteBeforeDate) as $id) { + $this->messageBus->dispatch(new RemoveOldVersionMessage($id)); + $maxDeleted = max($maxDeleted, $id); + } + + return [self::LAST_DELETED_KEY => $maxDeleted]; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessage.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessage.php new file mode 100644 index 000000000..a151e714c --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessage.php @@ -0,0 +1,19 @@ +logger->info(self::LOG_PREFIX.'Received one message', ['storedObjectVersionId' => $message->storedObjectVersionId]); + + $storedObjectVersion = $this->storedObjectVersionRepository->find($message->storedObjectVersionId); + + if (null === $storedObjectVersion) { + $this->logger->error(self::LOG_PREFIX.'StoredObjectVersion not found in database', ['storedObjectVersionId' => $message->storedObjectVersionId]); + throw new \RuntimeException('StoredObjectVersion not found with id '.$message->storedObjectVersionId); + } + + $this->storedObjectManager->delete($storedObjectVersion); + + $this->entityManager->remove($storedObjectVersion); + $this->entityManager->flush(); + + // clear the entity manager for future usage + $this->entityManager->clear(); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php index 2266bb0e2..51f32b98a 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php @@ -217,6 +217,23 @@ final class StoredObjectManager implements StoredObjectManagerInterface return $version; } + /** + * @throws StoredObjectManagerException + */ + public function delete(StoredObjectVersion $storedObjectVersion): void + { + $signedUrl = $this->tempUrlGenerator->generate('DELETE', $storedObjectVersion->getFilename()); + + try { + $response = $this->client->request('DELETE', $signedUrl->url); + if (Response::HTTP_NO_CONTENT !== $response->getStatusCode()) { + throw StoredObjectManagerException::invalidStatusCode($response->getStatusCode()); + } + } catch (TransportExceptionInterface $exception) { + throw StoredObjectManagerException::errorDuringHttpRequest($exception); + } + } + public function clearCache(): void { $this->inMemory = []; diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php index 4d2f45c33..7d4711b06 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php @@ -51,6 +51,11 @@ interface StoredObjectManagerInterface */ public function write(StoredObject $document, string $clearContent, ?string $contentType = null): StoredObjectVersion; + /** + * @throws StoredObjectManagerException + */ + public function delete(StoredObjectVersion $storedObjectVersion): void; + /** * return or compute the etag for the document. * diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/Repository/StoredObjectVersionRepositoryTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/Repository/StoredObjectVersionRepositoryTest.php new file mode 100644 index 000000000..ace122bea --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/Repository/StoredObjectVersionRepositoryTest.php @@ -0,0 +1,43 @@ +entityManager = self::getContainer()->get(EntityManagerInterface::class); + } + + public function testFindIdsByVersionsOlderThanDateAndNotLastVersion(): void + { + $repository = new StoredObjectVersionRepository($this->entityManager); + + // get old version, to get a chance to get one + $actual = $repository->findIdsByVersionsOlderThanDateAndNotLastVersion(new \DateTimeImmutable('1970-01-01')); + + self::assertIsIterable($actual); + self::assertContainsOnly('int', $actual); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php new file mode 100644 index 000000000..f27727d20 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php @@ -0,0 +1,104 @@ +createMock(StoredObjectVersionRepository::class); + $clock = new MockClock(new \DateTimeImmutable('2024-01-01 00:00:00', new \DateTimeZone('+00:00'))); + + $cronJob = new RemoveOldVersionCronJob($clock, $this->buildMessageBus(), $repository); + + self::assertEquals($expected, $cronJob->canRun($cronJobExecution)); + } + + public function testRun(): void + { + // we create a clock in the future. This led us a chance to having stored object to delete + $clock = new MockClock(new \DateTimeImmutable('2024-01-01 00:00:00', new \DateTimeZone('+00:00'))); + $repository = $this->createMock(StoredObjectVersionRepository::class); + $repository->expects($this->once()) + ->method('findIdsByVersionsOlderThanDateAndNotLastVersion') + ->with(new \DateTime('2023-10-03 00:00:00', new \DateTimeZone('+00:00'))) + ->willReturnCallback(function ($arg) { + yield 1; + yield 3; + yield 2; + }) + ; + + $cronJob = new RemoveOldVersionCronJob($clock, $this->buildMessageBus(true), $repository); + + $results = $cronJob->run([]); + + self::assertArrayHasKey('last-deleted-stored-object-version-id', $results); + self::assertIsInt($results['last-deleted-stored-object-version-id']); + } + + public static function buildTestCanRunData(): iterable + { + yield [ + (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-31 00:00:00', new \DateTimeZone('+00:00'))), + true, + ]; + + yield [ + (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-30 23:59:59', new \DateTimeZone('+00:00'))), + true, + ]; + + yield [ + (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-31 00:00:01', new \DateTimeZone('+00:00'))), + false, + ]; + + yield [ + null, + true, + ]; + } + + private function buildMessageBus(bool $expectDistpatchAtLeastOnce = false): MessageBusInterface + { + $messageBus = $this->createMock(MessageBusInterface::class); + + $methodDispatch = match ($expectDistpatchAtLeastOnce) { + true => $messageBus->expects($this->atLeastOnce())->method('dispatch')->with($this->isInstanceOf(RemoveOldVersionMessage::class)), + false => $messageBus->method('dispatch'), + }; + + $methodDispatch->willReturnCallback(function (RemoveOldVersionMessage $message) { + return new Envelope($message); + }); + + return $messageBus; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php new file mode 100644 index 000000000..109453bb7 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php @@ -0,0 +1,51 @@ +registerVersion(); + $storedObjectVersionRepository = $this->createMock(StoredObjectVersionRepository::class); + $storedObjectVersionRepository->expects($this->once())->method('find') + ->with($this->identicalTo(1)) + ->willReturn($version); + + $entityManager = $this->createMock(EntityManagerInterface::class); + $entityManager->expects($this->once())->method('remove')->with($this->identicalTo($version)); + $entityManager->expects($this->once())->method('flush'); + $entityManager->expects($this->once())->method('clear'); + + $storedObjectManager = $this->createMock(StoredObjectManagerInterface::class); + $storedObjectManager->expects($this->once())->method('delete')->with($this->identicalTo($version)); + + $handler = new RemoveOldVersionMessageHandler($storedObjectVersionRepository, new NullLogger(), $entityManager, $storedObjectManager); + + $handler(new RemoveOldVersionMessage(1)); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php index 3725f910f..4bb370ba3 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php @@ -198,6 +198,37 @@ final class StoredObjectManagerTest extends TestCase self::assertSame($storedObject->getCurrentVersion(), $newVersion); } + public function testDelete(): void + { + $storedObject = new StoredObject(); + $version = $storedObject->registerVersion(filename: 'object_name'); + + $httpClient = new MockHttpClient(function ($method, $url, $options) { + self::assertEquals('DELETE', $method); + self::assertEquals('https://example.com/object_name', $url); + + return new MockResponse('', [ + 'http_code' => 204, + ]); + }); + + $tempUrlGenerator = $this->createMock(TempUrlGeneratorInterface::class); + $tempUrlGenerator + ->expects($this->once()) + ->method('generate') + ->with($this->identicalTo('DELETE'), $this->identicalTo('object_name')) + ->willReturnCallback(function (string $method, string $objectName) { + return new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours') + ); + }); + + $storedObjectManager = new StoredObjectManager($httpClient, $tempUrlGenerator); + $storedObjectManager->delete($version); + } + public function testWriteWithDeleteAt() { $storedObject = new StoredObject(); From 0db2652f08008f1f5d5dd4eedf76f56710eb61e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 28 Aug 2024 11:41:43 +0200 Subject: [PATCH 12/25] Add cron job for removing expired stored objects Introduced `RemoveExpiredStoredObjectCronJob` to automate the deletion of expired stored objects every 7 days. Enhanced associated tests and updated relevant interfaces and classes to support the new cron job functionality. --- .../Entity/StoredObject.php | 27 +++- .../Entity/StoredObjectVersion.php | 23 +++- .../Repository/StoredObjectRepository.php | 11 ++ .../StoredObjectRepositoryInterface.php | 8 +- .../RemoveExpiredStoredObjectCronJob.php | 65 ++++++++++ .../RemoveOldVersionMessageHandler.php | 20 ++- .../Service/StoredObjectManager.php | 4 +- .../Service/StoredObjectManagerInterface.php | 9 +- .../RemoveExpiredStoredObjectCronJobTest.php | 115 ++++++++++++++++++ .../RemoveOldVersionMessageHandlerTest.php | 74 ++++++++++- 10 files changed, 344 insertions(+), 12 deletions(-) create mode 100644 src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJob.php create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJobTest.php diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php index 4fec9f6ca..fe68ec31c 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php @@ -90,7 +90,7 @@ class StoredObject implements Document, TrackCreationInterface /** * @var Collection */ - #[ORM\OneToMany(targetEntity: StoredObjectVersion::class, cascade: ['persist'], mappedBy: 'storedObject')] + #[ORM\OneToMany(targetEntity: StoredObjectVersion::class, cascade: ['persist'], mappedBy: 'storedObject', orphanRemoval: true)] private Collection $versions; /** @@ -333,6 +333,15 @@ class StoredObject implements Document, TrackCreationInterface return $version; } + public function removeVersion(StoredObjectVersion $storedObjectVersion): void + { + if (!$this->versions->contains($storedObjectVersion)) { + throw new \UnexpectedValueException('This stored object does not contains this version'); + } + $this->versions->removeElement($storedObjectVersion); + $storedObjectVersion->resetStoredObject(); + } + /** * @deprecated */ @@ -359,4 +368,20 @@ class StoredObject implements Document, TrackCreationInterface return uniqid(more_entropy: true); } } + + /** + * Checks if a stored object can be deleted. + * + * Currently, return true if the deletedAt date is below the current date, and the object + * does not contains any version (which must be removed first). + * + * @param \DateTimeImmutable $now the current date and time + * @param StoredObject $storedObject the stored object to check + * + * @return bool returns true if the stored object can be deleted, false otherwise + */ + public static function canBeDeleted(\DateTimeImmutable $now, StoredObject $storedObject): bool + { + return $storedObject->getDeleteAt() < $now && $storedObject->getVersions()->isEmpty(); + } } diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php index 2b8852513..84b10688f 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php @@ -33,6 +33,13 @@ class StoredObjectVersion implements TrackCreationInterface #[ORM\Column(type: \Doctrine\DBAL\Types\Types::INTEGER)] private ?int $id = null; + /** + * The stored object associated with this version. + */ + #[ORM\ManyToOne(targetEntity: StoredObject::class, inversedBy: 'versions')] + #[ORM\JoinColumn(name: 'stored_object_id', nullable: true)] + private ?StoredObject $storedObject; + /** * filename of the version in the stored object. */ @@ -40,12 +47,7 @@ class StoredObjectVersion implements TrackCreationInterface private string $filename = ''; public function __construct( - /** - * The stored object associated with this version. - */ - #[ORM\ManyToOne(targetEntity: StoredObject::class, inversedBy: 'versions')] - #[ORM\JoinColumn(name: 'stored_object_id', nullable: true)] - private StoredObject $storedObject, + StoredObject $storedObject, /** * The incremental version. @@ -76,6 +78,7 @@ class StoredObjectVersion implements TrackCreationInterface private string $type = '', ?string $filename = null, ) { + $this->storedObject = $storedObject; $this->filename = $filename ?? self::generateFilename($this); } @@ -124,4 +127,12 @@ class StoredObjectVersion implements TrackCreationInterface { return $this->version; } + + /** + * @internal to be used by StoredObject::removeVersion + */ + public function resetStoredObject(): void + { + $this->storedObject = null; + } } diff --git a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepository.php b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepository.php index 84bc7d4cb..2f39f2021 100644 --- a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepository.php +++ b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepository.php @@ -14,6 +14,7 @@ namespace Chill\DocStoreBundle\Repository; use Chill\DocStoreBundle\Entity\StoredObject; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\EntityRepository; +use Doctrine\ORM\Query; final readonly class StoredObjectRepository implements StoredObjectRepositoryInterface { @@ -53,6 +54,16 @@ final readonly class StoredObjectRepository implements StoredObjectRepositoryInt return $this->repository->findOneBy($criteria); } + public function findByExpired(\DateTimeImmutable $expiredAtDate): iterable + { + $qb = $this->repository->createQueryBuilder('stored_object'); + $qb + ->where('stored_object.deleteAt <= :expiredAt') + ->setParameter('expiredAt', $expiredAtDate); + + return $qb->getQuery()->toIterable(hydrationMode: Query::HYDRATE_OBJECT); + } + public function getClassName(): string { return StoredObject::class; diff --git a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepositoryInterface.php b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepositoryInterface.php index c694f1e09..45dcfcf94 100644 --- a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepositoryInterface.php +++ b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepositoryInterface.php @@ -17,4 +17,10 @@ use Doctrine\Persistence\ObjectRepository; /** * @extends ObjectRepository */ -interface StoredObjectRepositoryInterface extends ObjectRepository {} +interface StoredObjectRepositoryInterface extends ObjectRepository +{ + /** + * @return iterable + */ + public function findByExpired(\DateTimeImmutable $expiredAtDate): iterable; +} diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJob.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJob.php new file mode 100644 index 000000000..1dbf94080 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJob.php @@ -0,0 +1,65 @@ +clock->now() >= $cronJobExecution->getLastEnd()->add(new \DateInterval('P7D')); + } + + public function getKey(): string + { + return self::KEY; + } + + public function run(array $lastExecutionData): ?array + { + $lastDeleted = $lastExecutionData[self::LAST_DELETED_KEY] ?? 0; + + foreach ($this->storedObjectRepository->findByExpired($this->clock->now()) as $storedObject) { + foreach ($storedObject->getVersions() as $version) { + $this->messageBus->dispatch(new RemoveOldVersionMessage($version->getId())); + } + $lastDeleted = max($lastDeleted, $storedObject->getId()); + } + + return [self::LAST_DELETED_KEY => $lastDeleted]; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessageHandler.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessageHandler.php index 3d48f0267..69c9f283d 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessageHandler.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessageHandler.php @@ -11,13 +11,24 @@ declare(strict_types=1); namespace Chill\DocStoreBundle\Service\StoredObjectCleaner; +use Chill\DocStoreBundle\Entity\StoredObject; use Chill\DocStoreBundle\Exception\StoredObjectManagerException; use Chill\DocStoreBundle\Repository\StoredObjectVersionRepository; use Chill\DocStoreBundle\Service\StoredObjectManagerInterface; use Doctrine\ORM\EntityManagerInterface; use Psr\Log\LoggerInterface; +use Symfony\Component\Clock\ClockInterface; use Symfony\Component\Messenger\Handler\MessageHandlerInterface; +/** + * Class RemoveOldVersionMessageHandler. + * + * This class is responsible for handling the RemoveOldVersionMessage. It implements the MessageHandlerInterface. + * It removes old versions of stored objects based on certain conditions. + * + * If a StoredObject is a candidate for deletion (is expired and no more version stored), it is also removed from the + * database. + */ final readonly class RemoveOldVersionMessageHandler implements MessageHandlerInterface { private const LOG_PREFIX = '[RemoveOldVersionMessageHandler] '; @@ -27,6 +38,7 @@ final readonly class RemoveOldVersionMessageHandler implements MessageHandlerInt private LoggerInterface $logger, private EntityManagerInterface $entityManager, private StoredObjectManagerInterface $storedObjectManager, + private ClockInterface $clock, ) {} /** @@ -37,6 +49,7 @@ final readonly class RemoveOldVersionMessageHandler implements MessageHandlerInt $this->logger->info(self::LOG_PREFIX.'Received one message', ['storedObjectVersionId' => $message->storedObjectVersionId]); $storedObjectVersion = $this->storedObjectVersionRepository->find($message->storedObjectVersionId); + $storedObject = $storedObjectVersion->getStoredObject(); if (null === $storedObjectVersion) { $this->logger->error(self::LOG_PREFIX.'StoredObjectVersion not found in database', ['storedObjectVersionId' => $message->storedObjectVersionId]); @@ -44,8 +57,13 @@ final readonly class RemoveOldVersionMessageHandler implements MessageHandlerInt } $this->storedObjectManager->delete($storedObjectVersion); - + // to ensure an immediate deletion $this->entityManager->remove($storedObjectVersion); + + if (StoredObject::canBeDeleted($this->clock->now(), $storedObject)) { + $this->entityManager->remove($storedObject); + } + $this->entityManager->flush(); // clear the entity manager for future usage diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php index 51f32b98a..f87c60f82 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php @@ -226,9 +226,11 @@ final class StoredObjectManager implements StoredObjectManagerInterface try { $response = $this->client->request('DELETE', $signedUrl->url); - if (Response::HTTP_NO_CONTENT !== $response->getStatusCode()) { + if (! (Response::HTTP_NO_CONTENT === $response->getStatusCode() || Response::HTTP_NOT_FOUND === $response->getStatusCode())) { throw StoredObjectManagerException::invalidStatusCode($response->getStatusCode()); } + + $storedObjectVersion->getStoredObject()->removeVersion($storedObjectVersion); } catch (TransportExceptionInterface $exception) { throw StoredObjectManagerException::errorDuringHttpRequest($exception); } diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php index 7d4711b06..8dc6f27da 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php @@ -39,7 +39,9 @@ interface StoredObjectManagerInterface public function read(StoredObject|StoredObjectVersion $document): string; /** - * Set the content of a StoredObject. + * Register the content of a new version for the StoredObject. + * + * The manager is also responsible for registering a version in the StoredObject, and return this version. * * @param StoredObject $document the document * @param string $clearContent The content to store in clear @@ -52,6 +54,11 @@ interface StoredObjectManagerInterface public function write(StoredObject $document, string $clearContent, ?string $contentType = null): StoredObjectVersion; /** + * Remove a version from the storage. + * + * This method is also responsible for removing the version from the StoredObject (using @see{StoredObject::removeVersion}) + * in case of success. + * * @throws StoredObjectManagerException */ public function delete(StoredObjectVersion $storedObjectVersion): void; diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJobTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJobTest.php new file mode 100644 index 000000000..ed7341a38 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJobTest.php @@ -0,0 +1,115 @@ +createMock(StoredObjectRepositoryInterface::class); + $clock = new MockClock(new \DateTimeImmutable('2024-01-01 00:00:00', new \DateTimeZone('+00:00'))); + + $cronJob = new RemoveExpiredStoredObjectCronJob($clock, $this->buildMessageBus(), $repository); + + self::assertEquals($expected, $cronJob->canRun($cronJobExecution)); + } + + public static function buildTestCanRunData(): iterable + { + yield [ + (new CronJobExecution('remove-expired-stored-object'))->setLastEnd(new \DateTimeImmutable('2023-12-25 00:00:00', new \DateTimeZone('+00:00'))), + true, + ]; + + yield [ + (new CronJobExecution('remove-expired-stored-object'))->setLastEnd(new \DateTimeImmutable('2023-12-24 23:59:59', new \DateTimeZone('+00:00'))), + true, + ]; + + yield [ + (new CronJobExecution('remove-expired-stored-object'))->setLastEnd(new \DateTimeImmutable('2023-12-25 00:00:01', new \DateTimeZone('+00:00'))), + false, + ]; + + yield [ + null, + true, + ]; + } + + public function testRun(): void + { + $repository = $this->createMock(StoredObjectRepositoryInterface::class); + $repository->expects($this->atLeastOnce())->method('findByExpired')->withAnyParameters()->willReturnCallback( + function (\DateTimeImmutable $date): iterable { + yield $this->buildStoredObject(3); + yield $this->buildStoredObject(1); + } + ); + $clock = new MockClock(); + + $cronJob = new RemoveExpiredStoredObjectCronJob($clock, $this->buildMessageBus(true), $repository); + + $actual = $cronJob->run([]); + + self::assertEquals(3, $actual['last-deleted-stored-object-id']); + } + + private function buildStoredObject(int $id): StoredObject + { + $object = new StoredObject(); + $object->registerVersion(); + $class = new \ReflectionClass($object); + $idProperty = $class->getProperty('id'); + $idProperty->setValue($object, $id); + + $classVersion = new \ReflectionClass($object->getCurrentVersion()); + $idPropertyVersion = $classVersion->getProperty('id'); + $idPropertyVersion->setValue($object->getCurrentVersion(), $id); + + return $object; + } + + private function buildMessageBus(bool $expectDistpatchAtLeastOnce = false): MessageBusInterface + { + $messageBus = $this->createMock(MessageBusInterface::class); + + $methodDispatch = match ($expectDistpatchAtLeastOnce) { + true => $messageBus->expects($this->atLeastOnce())->method('dispatch')->with($this->isInstanceOf(RemoveOldVersionMessage::class)), + false => $messageBus->method('dispatch'), + }; + + $methodDispatch->willReturnCallback(function (RemoveOldVersionMessage $message) { + return new Envelope($message); + }); + + return $messageBus; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php index 109453bb7..be268225d 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php @@ -12,6 +12,7 @@ declare(strict_types=1); namespace Tests\Service\StoredObjectCleaner; use Chill\DocStoreBundle\Entity\StoredObject; +use Chill\DocStoreBundle\Entity\StoredObjectVersion; use Chill\DocStoreBundle\Repository\StoredObjectVersionRepository; use Chill\DocStoreBundle\Service\StoredObjectCleaner\RemoveOldVersionMessage; use Chill\DocStoreBundle\Service\StoredObjectCleaner\RemoveOldVersionMessageHandler; @@ -19,6 +20,7 @@ use Chill\DocStoreBundle\Service\StoredObjectManagerInterface; use Doctrine\ORM\EntityManagerInterface; use PHPUnit\Framework\TestCase; use Psr\Log\NullLogger; +use Symfony\Component\Clock\MockClock; /** * @internal @@ -44,8 +46,78 @@ class RemoveOldVersionMessageHandlerTest extends TestCase $storedObjectManager = $this->createMock(StoredObjectManagerInterface::class); $storedObjectManager->expects($this->once())->method('delete')->with($this->identicalTo($version)); - $handler = new RemoveOldVersionMessageHandler($storedObjectVersionRepository, new NullLogger(), $entityManager, $storedObjectManager); + $handler = new RemoveOldVersionMessageHandler($storedObjectVersionRepository, new NullLogger(), $entityManager, $storedObjectManager, new MockClock()); $handler(new RemoveOldVersionMessage(1)); } + + public function testInvokeWithStoredObjectToDelete(): void + { + $object = new StoredObject(); + $object->setDeleteAt(new \DateTimeImmutable('2023-12-01')); + $version = $object->registerVersion(); + + $storedObjectVersionRepository = $this->createMock(StoredObjectVersionRepository::class); + $storedObjectVersionRepository->expects($this->once())->method('find') + ->with($this->identicalTo(1)) + ->willReturn($version); + + $entityManager = $this->createMock(EntityManagerInterface::class); + $entityManager->expects($this->exactly(2))->method('remove')->with( + $this->logicalOr($this->identicalTo($version), $this->identicalTo($object)) + ); + $entityManager->expects($this->once())->method('flush'); + $entityManager->expects($this->once())->method('clear'); + + $handler = new RemoveOldVersionMessageHandler( + $storedObjectVersionRepository, + new NullLogger(), + $entityManager, + new DummyStoredObjectManager(), + new MockClock(new \DateTimeImmutable('2024-01-01')) + ); + + $handler(new RemoveOldVersionMessage(1)); + + self::assertCount(0, $object->getVersions()); + } +} + +class DummyStoredObjectManager implements StoredObjectManagerInterface +{ + public function getLastModified(StoredObject|StoredObjectVersion $document): \DateTimeInterface + { + throw new \RuntimeException(); + } + + public function getContentLength(StoredObject|StoredObjectVersion $document): int + { + throw new \RuntimeException(); + } + + public function read(StoredObject|StoredObjectVersion $document): string + { + throw new \RuntimeException(); + } + + public function write(StoredObject $document, string $clearContent, ?string $contentType = null): StoredObjectVersion + { + throw new \RuntimeException(); + } + + public function delete(StoredObjectVersion $storedObjectVersion): void + { + $object = $storedObjectVersion->getStoredObject(); + $object->removeVersion($storedObjectVersion); + } + + public function etag(StoredObject|StoredObjectVersion $document): string + { + throw new \RuntimeException(); + } + + public function clearCache(): void + { + throw new \RuntimeException(); + } } From e477a49c92afdeab0719f21723098f5db1732f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 28 Aug 2024 15:26:19 +0200 Subject: [PATCH 13/25] Integrate SwaggerUI with direct download and cleanup package Added a customized SwaggerUI HTML template under Dev directory. Removed outdated swagger-ui dependency from package.json to streamline dependency management. --- package.json | 1 - .../views/Dev/swagger-ui/index.html.twig | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 src/Bundle/ChillMainBundle/Resources/views/Dev/swagger-ui/index.html.twig diff --git a/package.json b/package.json index e0417cd4f..f0d6ca062 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,6 @@ "masonry-layout": "^4.2.2", "mime": "^4.0.0", "pdfjs-dist": "^4.3.136", - "swagger-ui": "^4.15.5", "vis-network": "^9.1.0", "vue": "^3.2.37", "vue-i18n": "^9.1.6", diff --git a/src/Bundle/ChillMainBundle/Resources/views/Dev/swagger-ui/index.html.twig b/src/Bundle/ChillMainBundle/Resources/views/Dev/swagger-ui/index.html.twig new file mode 100644 index 000000000..21d1ac505 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Resources/views/Dev/swagger-ui/index.html.twig @@ -0,0 +1,22 @@ + + + + + + + SwaggerUI + + + +
+ + + + From 2d82c1e1050c5a979e92451d2344d6faa7156a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 28 Aug 2024 15:32:57 +0200 Subject: [PATCH 14/25] rector fixes after rector's upgrade --- .../Entity/StoredObject.php | 2 +- .../StoredObjectVersionRepository.php | 4 +- .../RemoveExpiredStoredObjectCronJob.php | 8 +-- .../RemoveExpiredStoredObjectCronJobTest.php | 4 +- .../RemoveOldVersionCronJobTest.php | 4 +- .../Tests/Service/StoredObjectManagerTest.php | 52 ++++++++----------- 6 files changed, 31 insertions(+), 43 deletions(-) diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php index fe68ec31c..87ddd5785 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php @@ -364,7 +364,7 @@ class StoredObject implements Document, TrackCreationInterface { try { return base_convert(bin2hex(random_bytes(32)), 16, 36); - } catch (RandomException $e) { + } catch (RandomException) { return uniqid(more_entropy: true); } } diff --git a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php index 047b024a9..1ab9b9edd 100644 --- a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php +++ b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php @@ -23,9 +23,9 @@ use Doctrine\Persistence\ObjectRepository; */ class StoredObjectVersionRepository implements ObjectRepository { - private EntityRepository $repository; + private readonly EntityRepository $repository; - private Connection $connection; + private readonly Connection $connection; public function __construct(EntityManagerInterface $entityManager) { diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJob.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJob.php index 1dbf94080..62ae154bc 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJob.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJob.php @@ -23,16 +23,16 @@ use Symfony\Component\Messenger\MessageBusInterface; * This cronjob is executed every 7days, to remove expired stored object. For every * expired stored object, every version is sent to message bus for async deletion. */ -final class RemoveExpiredStoredObjectCronJob implements CronJobInterface +final readonly class RemoveExpiredStoredObjectCronJob implements CronJobInterface { public const string KEY = 'remove-expired-stored-object'; private const string LAST_DELETED_KEY = 'last-deleted-stored-object-id'; public function __construct( - private readonly ClockInterface $clock, - private readonly MessageBusInterface $messageBus, - private readonly StoredObjectRepositoryInterface $storedObjectRepository + private ClockInterface $clock, + private MessageBusInterface $messageBus, + private StoredObjectRepositoryInterface $storedObjectRepository ) {} public function canRun(?CronJobExecution $cronJobExecution): bool diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJobTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJobTest.php index ed7341a38..3d6d758fa 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJobTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveExpiredStoredObjectCronJobTest.php @@ -106,9 +106,7 @@ class RemoveExpiredStoredObjectCronJobTest extends TestCase false => $messageBus->method('dispatch'), }; - $methodDispatch->willReturnCallback(function (RemoveOldVersionMessage $message) { - return new Envelope($message); - }); + $methodDispatch->willReturnCallback(fn (RemoveOldVersionMessage $message) => new Envelope($message)); return $messageBus; } diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php index f27727d20..937253c81 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php @@ -95,9 +95,7 @@ class RemoveOldVersionCronJobTest extends KernelTestCase false => $messageBus->method('dispatch'), }; - $methodDispatch->willReturnCallback(function (RemoveOldVersionMessage $message) { - return new Envelope($message); - }); + $methodDispatch->willReturnCallback(fn (RemoveOldVersionMessage $message) => new Envelope($message)); return $messageBus; } diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php index 4bb370ba3..e09a2dc1e 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php @@ -217,13 +217,11 @@ final class StoredObjectManagerTest extends TestCase ->expects($this->once()) ->method('generate') ->with($this->identicalTo('DELETE'), $this->identicalTo('object_name')) - ->willReturnCallback(function (string $method, string $objectName) { - return new SignedUrl( - $method, - 'https://example.com/'.$objectName, - new \DateTimeImmutable('1 hours') - ); - }); + ->willReturnCallback(fn (string $method, string $objectName) => new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours') + )); $storedObjectManager = new StoredObjectManager($httpClient, $tempUrlGenerator); $storedObjectManager->delete($version); @@ -276,14 +274,12 @@ final class StoredObjectManagerTest extends TestCase ->expects($this->never()) ->method('generate') ->with($this->identicalTo('HEAD'), $this->isType('string')) - ->willReturnCallback(function (string $method, string $objectName) { - return new SignedUrl( - $method, - 'https://example.com/'.$objectName, - new \DateTimeImmutable('1 hours'), - $objectName - ); - }); + ->willReturnCallback(fn (string $method, string $objectName) => new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours'), + $objectName + )); $manager = new StoredObjectManager($client, $tempUrlGenerator); @@ -307,13 +303,11 @@ final class StoredObjectManagerTest extends TestCase ->expects($this->atLeastOnce()) ->method('generate') ->with($this->identicalTo('HEAD'), $this->isType('string')) - ->willReturnCallback(function (string $method, string $objectName) { - return new SignedUrl( - $method, - 'https://example.com/'.$objectName, - new \DateTimeImmutable('1 hours') - ); - }); + ->willReturnCallback(fn (string $method, string $objectName) => new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours') + )); $manager = new StoredObjectManager($client, $tempUrlGenerator); @@ -368,14 +362,12 @@ final class StoredObjectManagerTest extends TestCase ->expects($this->atLeastOnce()) ->method('generate') ->with($this->logicalOr($this->identicalTo('GET'), $this->identicalTo('PUT')), $this->isType('string')) - ->willReturnCallback(function (string $method, string $objectName) { - return new SignedUrl( - $method, - 'https://example.com/'.$objectName, - new \DateTimeImmutable('1 hours'), - $objectName - ); - }); + ->willReturnCallback(fn (string $method, string $objectName) => new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours'), + $objectName + )); return $tempUrlGenerator; } From 7ab52ff09e0f16bf42ad5dd77bc55bbc931f7388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 28 Aug 2024 15:34:42 +0200 Subject: [PATCH 15/25] Add stored object creation endpoint Introduced a new API endpoint to create stored objects with access control for roles 'ROLE_ADMIN' and 'ROLE_USER'. Updated corresponding routes, removed unused dependencies, and added unit tests to ensure functionality. --- .../Controller/StoredObjectApiController.php | 42 +++++++++++++- .../ChillDocStoreExtension.php | 25 --------- .../StoredObjectApiControllerTest.php | 55 +++++++++++++++++++ .../ChillDocStoreBundle/chill.api.specs.yaml | 9 +-- 4 files changed, 97 insertions(+), 34 deletions(-) create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/Controller/StoredObjectApiControllerTest.php diff --git a/src/Bundle/ChillDocStoreBundle/Controller/StoredObjectApiController.php b/src/Bundle/ChillDocStoreBundle/Controller/StoredObjectApiController.php index 600b6e52d..cf5830c53 100644 --- a/src/Bundle/ChillDocStoreBundle/Controller/StoredObjectApiController.php +++ b/src/Bundle/ChillDocStoreBundle/Controller/StoredObjectApiController.php @@ -11,6 +11,46 @@ declare(strict_types=1); namespace Chill\DocStoreBundle\Controller; +use Chill\DocStoreBundle\Entity\StoredObject; use Chill\MainBundle\CRUD\Controller\ApiController; +use Doctrine\ORM\EntityManagerInterface; +use Symfony\Component\HttpFoundation\JsonResponse; +use Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException; +use Symfony\Component\Routing\Annotation\Route; +use Symfony\Component\Security\Core\Security; +use Symfony\Component\Serializer\Normalizer\AbstractNormalizer; +use Symfony\Component\Serializer\SerializerInterface; -class StoredObjectApiController extends ApiController {} +class StoredObjectApiController extends ApiController +{ + public function __construct( + private readonly Security $security, + private readonly SerializerInterface $serializer, + private readonly EntityManagerInterface $entityManager + ) {} + + /** + * Creates a new stored object. + * + * @return JsonResponse the response containing the serialized object in JSON format + * + * @throws AccessDeniedHttpException if the user does not have the necessary role to create a stored object + */ + #[Route('/api/1.0/doc-store/stored-object/create', methods: ['POST'])] + public function createStoredObject(): JsonResponse + { + if (!($this->security->isGranted('ROLE_ADMIN') || $this->security->isGranted('ROLE_USER'))) { + throw new AccessDeniedHttpException('Must be user or admin to create a stored object'); + } + + $object = new StoredObject(); + + $this->entityManager->persist($object); + $this->entityManager->flush(); + + return new JsonResponse( + $this->serializer->serialize($object, 'json', [AbstractNormalizer::GROUPS => ['read']]), + json: true + ); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/DependencyInjection/ChillDocStoreExtension.php b/src/Bundle/ChillDocStoreBundle/DependencyInjection/ChillDocStoreExtension.php index b156af1ea..efeb6362d 100644 --- a/src/Bundle/ChillDocStoreBundle/DependencyInjection/ChillDocStoreExtension.php +++ b/src/Bundle/ChillDocStoreBundle/DependencyInjection/ChillDocStoreExtension.php @@ -11,7 +11,6 @@ declare(strict_types=1); namespace Chill\DocStoreBundle\DependencyInjection; -use Chill\DocStoreBundle\Controller\StoredObjectApiController; use Chill\DocStoreBundle\Security\Authorization\AccompanyingCourseDocumentVoter; use Chill\DocStoreBundle\Security\Authorization\PersonDocumentVoter; use Chill\DocStoreBundle\Security\Authorization\StoredObjectVoterInterface; @@ -19,7 +18,6 @@ use Symfony\Component\Config\FileLocator; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Extension\PrependExtensionInterface; use Symfony\Component\DependencyInjection\Loader; -use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpKernel\DependencyInjection\Extension; /** @@ -53,29 +51,6 @@ class ChillDocStoreExtension extends Extension implements PrependExtensionInterf $this->prependRoute($container); $this->prependAuthorization($container); $this->prependTwig($container); - $this->prependApis($container); - } - - protected function prependApis(ContainerBuilder $container) - { - $container->prependExtensionConfig('chill_main', [ - 'apis' => [ - [ - 'class' => \Chill\DocStoreBundle\Entity\StoredObject::class, - 'controller' => StoredObjectApiController::class, - 'name' => 'stored_object', - 'base_path' => '/api/1.0/docstore/stored-object', - 'base_role' => 'ROLE_USER', - 'actions' => [ - '_entity' => [ - 'methods' => [ - Request::METHOD_POST => true, - ], - ], - ], - ], - ], - ]); } protected function prependAuthorization(ContainerBuilder $container) diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Controller/StoredObjectApiControllerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Controller/StoredObjectApiControllerTest.php new file mode 100644 index 000000000..74dbd9b46 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Controller/StoredObjectApiControllerTest.php @@ -0,0 +1,55 @@ +createMock(Security::class); + $security->expects($this->atLeastOnce())->method('isGranted') + ->with($this->logicalOr($this->identicalTo('ROLE_ADMIN'), $this->identicalTo('ROLE_USER'))) + ->willReturn(true) + ; + + $entityManager = $this->createMock(EntityManagerInterface::class); + $entityManager->expects($this->once())->method('persist')->with($this->isInstanceOf(StoredObject::class)); + $entityManager->expects($this->once())->method('flush'); + + $serializer = $this->createMock(SerializerInterface::class); + $serializer->expects($this->once())->method('serialize') + ->with($this->isInstanceOf(StoredObject::class), 'json', $this->anything()) + ->willReturn($r = <<<'JSON' + {"type": "stored-object", "id": 1} + JSON); + + $controller = new StoredObjectApiController($security, $serializer, $entityManager); + + $actual = $controller->createStoredObject(); + + self::assertInstanceOf(JsonResponse::class, $actual); + self::assertEquals($r, $actual->getContent()); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/chill.api.specs.yaml b/src/Bundle/ChillDocStoreBundle/chill.api.specs.yaml index d74311387..7864a18af 100644 --- a/src/Bundle/ChillDocStoreBundle/chill.api.specs.yaml +++ b/src/Bundle/ChillDocStoreBundle/chill.api.specs.yaml @@ -21,18 +21,11 @@ components: type: string paths: - /1.0/docstore/stored-object.json: + /1.0/doc-store/stored-object/create: post: tags: - storedobject summary: Create a stored object - requestBody: - description: "A stored object" - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/StoredObject" responses: 200: description: "OK" From 00cc3b78060183a0b69007e735357c5370c9addb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 28 Aug 2024 18:00:20 +0200 Subject: [PATCH 16/25] Refactor backend for getting signed url --- .../TempUrlOpenstackGenerator.php | 5 +- .../AsyncUpload/TempUrlGeneratorInterface.php | 3 +- .../Controller/AsyncUploadController.php | 118 +++++++----- .../Entity/StoredObject.php | 7 +- .../Resources/public/types.ts | 38 ++-- .../Controller/AsyncUploadControllerTest.php | 179 +++++++++++++++--- .../ChillDocStoreBundle/chill.api.specs.yaml | 130 ++++++++++--- 7 files changed, 345 insertions(+), 135 deletions(-) diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php index ae74f9d0e..3ba1f0dea 100644 --- a/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php +++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php @@ -58,6 +58,7 @@ final readonly class TempUrlOpenstackGenerator implements TempUrlGeneratorInterf ?int $expire_delay = null, ?int $submit_delay = null, int $max_file_count = 1, + ?string $object_name = null, ): SignedUrlPost { $delay = $expire_delay ?? $this->max_expire_delay; $submit_delay ??= $this->max_submit_delay; @@ -84,7 +85,9 @@ final readonly class TempUrlOpenstackGenerator implements TempUrlGeneratorInterf $expires = $this->clock->now()->add(new \DateInterval('PT'.(string) $delay.'S')); - $object_name = $this->generateObjectName(); + if (null === $object_name) { + $object_name = $this->generateObjectName(); + } $g = new SignedUrlPost( $url = $this->generateUrl($object_name), diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/TempUrlGeneratorInterface.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/TempUrlGeneratorInterface.php index 08ac9e062..e6589b3c7 100644 --- a/src/Bundle/ChillDocStoreBundle/AsyncUpload/TempUrlGeneratorInterface.php +++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/TempUrlGeneratorInterface.php @@ -16,7 +16,8 @@ interface TempUrlGeneratorInterface public function generatePost( ?int $expire_delay = null, ?int $submit_delay = null, - int $max_file_count = 1 + int $max_file_count = 1, + ?string $object_name = null, ): SignedUrlPost; public function generate(string $method, string $object_name, ?int $expire_delay = null): SignedUrl; diff --git a/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php b/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php index 75be20d34..879230cd9 100644 --- a/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php +++ b/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php @@ -11,9 +11,10 @@ declare(strict_types=1); namespace Chill\DocStoreBundle\Controller; -use Chill\DocStoreBundle\AsyncUpload\Exception\TempUrlGeneratorException; use Chill\DocStoreBundle\AsyncUpload\TempUrlGeneratorInterface; -use Chill\DocStoreBundle\Security\Authorization\AsyncUploadVoter; +use Chill\DocStoreBundle\Entity\StoredObject; +use Chill\DocStoreBundle\Entity\StoredObjectVersion; +use Chill\DocStoreBundle\Security\Authorization\StoredObjectRoleEnum; use Psr\Log\LoggerInterface; use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\HttpFoundation\Request; @@ -30,62 +31,77 @@ final readonly class AsyncUploadController private TempUrlGeneratorInterface $tempUrlGenerator, private SerializerInterface $serializer, private Security $security, - private LoggerInterface $logger, + private LoggerInterface $chillLogger, ) {} - #[Route(path: '/asyncupload/temp_url/generate/{method}', name: 'async_upload.generate_url')] - public function getSignedUrl(string $method, Request $request): JsonResponse + #[Route(path: '/api/1.0/doc-store/async-upload/temp_url/{uuid}/generate/post', name: 'chill_docstore_asyncupload_getsignedurlpost')] + public function getSignedUrlPost(Request $request, StoredObject $storedObject): JsonResponse { - try { - switch (strtolower($method)) { - case 'post': - $p = $this->tempUrlGenerator - ->generatePost( - $request->query->has('expires_delay') ? $request->query->getInt('expires_delay') : null, - $request->query->has('submit_delay') ? $request->query->getInt('submit_delay') : null - ) - ; - break; - case 'get': - case 'head': - $object_name = $request->query->get('object_name', null); + if (!$this->security->isGranted(StoredObjectRoleEnum::EDIT->value, $storedObject)) { + throw new AccessDeniedHttpException('not able to edit the given stored object'); + } - if (null === $object_name) { - return (new JsonResponse((object) [ - 'message' => 'the object_name is null', - ])) - ->setStatusCode(JsonResponse::HTTP_BAD_REQUEST); - } - $p = $this->tempUrlGenerator->generate( - $method, - $object_name, - $request->query->has('expires_delay') ? $request->query->getInt('expires_delay') : null - ); - break; - default: - return (new JsonResponse((object) ['message' => 'the method ' - ."{$method} is not valid"])) - ->setStatusCode(JsonResponse::HTTP_BAD_REQUEST); + // we create a dummy version, to generate a filename + $version = $storedObject->registerVersion(); + + $p = $this->tempUrlGenerator + ->generatePost( + $request->query->has('expires_delay') ? $request->query->getInt('expires_delay') : null, + $request->query->has('submit_delay') ? $request->query->getInt('submit_delay') : null, + object_name: $version->getFilename() + ); + + $this->chillLogger->notice('[Privacy Event] a request to upload a document has been generated', [ + 'doc_uuid' => $storedObject->getUuid(), + ]); + + return new JsonResponse( + $this->serializer->serialize($p, 'json', [AbstractNormalizer::GROUPS => ['read']]), + Response::HTTP_OK, + [], + true + ); + } + + #[Route(path: '/api/1.0/doc-store/async-upload/temp_url/{uuid}/generate/{method}', name: 'chill_docstore_asyncupload_getsignedurlget', requirements: ['method' => 'get|head'])] + public function getSignedUrlGet(Request $request, StoredObject $storedObject, string $method): JsonResponse + { + if (!$this->security->isGranted(StoredObjectRoleEnum::SEE->value, $storedObject)) { + throw new AccessDeniedHttpException('not able to read the given stored object'); + } + + // we really want to be sure that there are no other method than get or head: + if (!in_array($method, ['get', 'head'], true)) { + throw new AccessDeniedHttpException('Only methods get and head are allowed'); + } + + if ($request->query->has('version')) { + $filename = $request->query->get('version'); + + $storedObjectVersion = $storedObject->getVersions()->findFirst(fn(int $index, StoredObjectVersion $version): bool => $version->getFilename() === $filename); + + if (null === $storedObjectVersion) { + // we are here in the case where the version is not stored into the database + // as the version is prefixed by the stored object prefix, we just have to check that this prefix + // is the same. It means that the user had previously the permission to "SEE_AND_EDIT" this stored + // object with same prefix that we checked before + if (!str_starts_with($filename, $storedObject->getPrefix())) { + throw new AccessDeniedHttpException('not able to match the version with the same filename'); + } } - } catch (TempUrlGeneratorException $e) { - $this->logger->warning('The client requested a temp url' - .' which sparkle an error.', [ - 'message' => $e->getMessage(), - 'expire_delay' => $request->query->getInt('expire_delay', 0), - 'file_count' => $request->query->getInt('file_count', 1), - 'method' => $method, - ]); - - $p = new \stdClass(); - $p->message = $e->getMessage(); - $p->status = JsonResponse::HTTP_BAD_REQUEST; - - return new JsonResponse($p, JsonResponse::HTTP_BAD_REQUEST); + } else { + $filename = $storedObject->getCurrentVersion()->getFilename(); } - if (!$this->security->isGranted(AsyncUploadVoter::GENERATE_SIGNATURE, $p)) { - throw new AccessDeniedHttpException('not allowed to generate this signature'); - } + $p = $this->tempUrlGenerator->generate( + $method, + $filename, + $request->query->has('expires_delay') ? $request->query->getInt('expires_delay') : null + ); + + $this->chillLogger->notice('[Privacy Event] a request to see a document has been granted', [ + 'doc_uuid' => $storedObject->getUuid(), + ]); return new JsonResponse( $this->serializer->serialize($p, 'json', [AbstractNormalizer::GROUPS => ['read']]), diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php index 87ddd5785..6ed52716d 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php @@ -41,6 +41,7 @@ use Symfony\Component\Serializer\Annotation as Serializer; class StoredObject implements Document, TrackCreationInterface { use TrackCreationTrait; + final public const STATUS_EMPTY = 'empty'; final public const STATUS_READY = 'ready'; final public const STATUS_PENDING = 'pending'; final public const STATUS_FAILURE = 'failure'; @@ -98,7 +99,7 @@ class StoredObject implements Document, TrackCreationInterface */ public function __construct( #[ORM\Column(type: \Doctrine\DBAL\Types\Types::TEXT, options: ['default' => 'ready'])] - private string $status = 'ready' + private string $status = 'empty' ) { $this->uuid = Uuid::uuid4(); $this->versions = new ArrayCollection(); @@ -330,6 +331,10 @@ class StoredObject implements Document, TrackCreationInterface $this->versions->add($version); + if ('empty' === $this->status) { + $this->status = self::STATUS_READY; + } + return $version; } diff --git a/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts b/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts index 1fc8b1cdd..1d21feacd 100644 --- a/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts +++ b/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts @@ -1,27 +1,27 @@ import {DateTime} from "../../../ChillMainBundle/Resources/public/types"; -export type StoredObjectStatus = "ready"|"failure"|"pending"; +export type StoredObjectStatus = "empty"|"ready"|"failure"|"pending"; export interface StoredObject { - id: number, + id: number, - /** - * filename of the object in the object storage - */ - filename: string, - creationDate: DateTime, - datas: object, - iv: number[], - keyInfos: object, - title: string, - type: string, - uuid: string, - status: StoredObjectStatus, + /** + * filename of the object in the object storage + */ + filename: string, + creationDate: DateTime, + datas: object, + iv: number[], + keyInfos: object, + title: string, + type: string, + uuid: string, + status: StoredObjectStatus, _links?: { - dav_link?: { - href: string - expiration: number - }, + dav_link?: { + href: string + expiration: number + }, } } @@ -82,4 +82,4 @@ export interface Signature { zones: SignatureZone[], } -export type SignedState = 'pending' | 'signed' | 'rejected' | 'canceled' | 'error'; \ No newline at end of file +export type SignedState = 'pending' | 'signed' | 'rejected' | 'canceled' | 'error'; diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Controller/AsyncUploadControllerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Controller/AsyncUploadControllerTest.php index 6b49b2919..489bf1185 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Controller/AsyncUploadControllerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Controller/AsyncUploadControllerTest.php @@ -15,7 +15,7 @@ use Chill\DocStoreBundle\AsyncUpload\SignedUrl; use Chill\DocStoreBundle\AsyncUpload\SignedUrlPost; use Chill\DocStoreBundle\AsyncUpload\TempUrlGeneratorInterface; use Chill\DocStoreBundle\Controller\AsyncUploadController; -use Chill\DocStoreBundle\Security\Authorization\AsyncUploadVoter; +use Chill\DocStoreBundle\Entity\StoredObject; use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; @@ -34,46 +34,165 @@ class AsyncUploadControllerTest extends TestCase { use ProphecyTrait; - public function testGenerateWhenUserIsNotGranted(): void + public function testGetSignedUrlPost(): void { - $this->expectException(AccessDeniedHttpException::class); - $controller = $this->buildAsyncUploadController(false); + $storedObject = new StoredObject(); + $security = $this->prophesize(Security::class); + $security->isGranted('SEE_AND_EDIT', $storedObject)->willReturn(true)->shouldBeCalled(); - $controller->getSignedUrl('POST', new Request()); - } + $controller = new AsyncUploadController( + $this->buildTempUrlGenerator(), + $this->buildSerializer(), + $security->reveal(), + new NullLogger(), + ); - public function testGeneratePost(): void - { - $controller = $this->buildAsyncUploadController(true); + $actual = $controller->getSignedUrlPost(new Request(query: ['expires_delay' => 10, 'submit_delay' => 1800]), $storedObject); - $actual = $controller->getSignedUrl('POST', new Request()); $decodedActual = json_decode($actual->getContent(), true, JSON_THROW_ON_ERROR, JSON_THROW_ON_ERROR); self::assertArrayHasKey('method', $decodedActual); self::assertEquals('POST', $decodedActual['method']); } - public function testGenerateGet(): void + public function testGetSignedUrlGetSimpleScenarioHappy(): void { - $controller = $this->buildAsyncUploadController(true); + $storedObject = new StoredObject(); + $storedObject->registerVersion(); + $security = $this->prophesize(Security::class); + $security->isGranted('SEE', $storedObject)->willReturn(true)->shouldBeCalled(); + + $controller = new AsyncUploadController( + $this->buildTempUrlGenerator(), + $this->buildSerializer(), + $security->reveal(), + new NullLogger(), + ); + + $actual = $controller->getSignedUrlGet(new Request(query: ['expires_delay' => 10, 'submit_delay' => 1800]), $storedObject, 'get'); - $actual = $controller->getSignedUrl('GET', new Request(['object_name' => 'abc'])); $decodedActual = json_decode($actual->getContent(), true, JSON_THROW_ON_ERROR, JSON_THROW_ON_ERROR); self::assertArrayHasKey('method', $decodedActual); self::assertEquals('GET', $decodedActual['method']); } - private function buildAsyncUploadController( - bool $isGranted, - ): AsyncUploadController { - $tempUrlGenerator = new class () implements TempUrlGeneratorInterface { - public function generatePost(?int $expire_delay = null, ?int $submit_delay = null, int $max_file_count = 1): SignedUrlPost + public function testGetSignedUrlGetSimpleScenarioNotAuthorized(): void + { + $this->expectException(AccessDeniedHttpException::class); + + $storedObject = new StoredObject(); + $storedObject->registerVersion(); + $security = $this->prophesize(Security::class); + $security->isGranted('SEE', $storedObject)->willReturn(false)->shouldBeCalled(); + + $controller = new AsyncUploadController( + $this->buildTempUrlGenerator(), + $this->buildSerializer(), + $security->reveal(), + new NullLogger(), + ); + + $controller->getSignedUrlGet(new Request(query: ['expires_delay' => 10, 'submit_delay' => 1800]), $storedObject, 'get'); + } + + public function testGetSignedUrlGetForSpecificVersionOfTheStoredObjectStoredInDatabase(): void + { + $storedObject = new StoredObject(); + $version = $storedObject->registerVersion(); + // we add a version to be sure that the we do not get the last one + $storedObject->registerVersion(); + + $security = $this->prophesize(Security::class); + $security->isGranted('SEE', $storedObject)->willReturn(true)->shouldBeCalled(); + + $controller = new AsyncUploadController( + $this->buildTempUrlGenerator(), + $this->buildSerializer(), + $security->reveal(), + new NullLogger(), + ); + + $actual = $controller->getSignedUrlGet( + new Request(query: ['expires_delay' => 10, 'submit_delay' => 1800, 'version' => $version->getFilename()]), + $storedObject, + 'get' + ); + + $decodedActual = json_decode($actual->getContent(), true, JSON_THROW_ON_ERROR, JSON_THROW_ON_ERROR); + + self::assertArrayHasKey('method', $decodedActual); + self::assertEquals('GET', $decodedActual['method']); + self::assertArrayHasKey('object_name', $decodedActual); + self::assertEquals($version->getFilename(), $decodedActual['object_name']); + } + + public function testGetSignedUrlGetForSpecificVersionOfTheStoredObjectNotYetStoredInDatabase(): void + { + $storedObject = new StoredObject(); + $storedObject->registerVersion(); + // we generate a valid name + $version = $storedObject->getPrefix().'/some-version'; + + $security = $this->prophesize(Security::class); + $security->isGranted('SEE', $storedObject)->willReturn(true)->shouldBeCalled(); + + $controller = new AsyncUploadController( + $this->buildTempUrlGenerator(), + $this->buildSerializer(), + $security->reveal(), + new NullLogger(), + ); + + $actual = $controller->getSignedUrlGet( + new Request(query: ['expires_delay' => 10, 'submit_delay' => 1800, 'version' => $version]), + $storedObject, + 'get' + ); + + $decodedActual = json_decode($actual->getContent(), true, JSON_THROW_ON_ERROR, JSON_THROW_ON_ERROR); + + self::assertArrayHasKey('method', $decodedActual); + self::assertEquals('GET', $decodedActual['method']); + self::assertArrayHasKey('object_name', $decodedActual); + self::assertEquals($version, $decodedActual['object_name']); + } + + public function testGetSignedUrlGetForSpecificVersionNotBelongingToTheStoreObject(): void + { + $this->expectException(AccessDeniedHttpException::class); + + $storedObject = new StoredObject(); + $storedObject->registerVersion(); + // we generate a random version + $version = 'something/else'; + + $security = $this->prophesize(Security::class); + $security->isGranted('SEE', $storedObject)->willReturn(true)->shouldBeCalled(); + + $controller = new AsyncUploadController( + $this->buildTempUrlGenerator(), + $this->buildSerializer(), + $security->reveal(), + new NullLogger(), + ); + + $controller->getSignedUrlGet( + new Request(query: ['expires_delay' => 10, 'submit_delay' => 1800, 'version' => $version]), + $storedObject, + 'get' + ); + } + + public function buildTempUrlGenerator(): TempUrlGeneratorInterface + { + return new class () implements TempUrlGeneratorInterface { + public function generatePost(?int $expire_delay = null, ?int $submit_delay = null, int $max_file_count = 1, ?string $object_name = null): SignedUrlPost { return new SignedUrlPost( 'https://object.store.example', new \DateTimeImmutable('1 hour'), - 'abc', + $object_name ?? 'abc', 150, 1, 1800, @@ -86,27 +205,25 @@ class AsyncUploadControllerTest extends TestCase public function generate(string $method, string $object_name, ?int $expire_delay = null): SignedUrl { return new SignedUrl( - $method, + strtoupper($method), 'https://object.store.example', new \DateTimeImmutable('1 hour'), $object_name ); } }; + } + private function buildSerializer(): SerializerInterface + { $serializer = $this->prophesize(SerializerInterface::class); $serializer->serialize(Argument::type(SignedUrl::class), 'json', Argument::type('array')) - ->will(fn (array $args): string => json_encode(['method' => $args[0]->method], JSON_THROW_ON_ERROR, 3)); + ->will(fn (array $args): string => json_encode( + ['method' => $args[0]->method, 'object_name' => $args[0]->object_name], + JSON_THROW_ON_ERROR, + 3 + )); - $security = $this->prophesize(Security::class); - $security->isGranted(AsyncUploadVoter::GENERATE_SIGNATURE, Argument::type(SignedUrl::class)) - ->willReturn($isGranted); - - return new AsyncUploadController( - $tempUrlGenerator, - $serializer->reveal(), - $security->reveal(), - new NullLogger() - ); + return $serializer->reveal(); } } diff --git a/src/Bundle/ChillDocStoreBundle/chill.api.specs.yaml b/src/Bundle/ChillDocStoreBundle/chill.api.specs.yaml index 7864a18af..28345e463 100644 --- a/src/Bundle/ChillDocStoreBundle/chill.api.specs.yaml +++ b/src/Bundle/ChillDocStoreBundle/chill.api.specs.yaml @@ -1,39 +1,107 @@ --- openapi: "3.0.0" info: - version: "1.0.0" - title: "Chill api" - description: "Api documentation for chill. Currently, work in progress" + version: "1.0.0" + title: "Chill api" + description: "Api documentation for chill. Currently, work in progress" servers: - - url: "/api" - description: "Your current dev server" + - url: "/api" + description: "Your current dev server" components: - schemas: - StoredObject: - type: object - properties: - id: - type: integer - filename: - type: string - type: - type: string + schemas: + StoredObject: + type: object + properties: + id: + type: integer + filename: + type: string + type: + type: string paths: - /1.0/doc-store/stored-object/create: - post: - tags: - - storedobject - summary: Create a stored object - responses: - 200: - description: "OK" - content: - application/json: - schema: - $ref: "#/components/schemas/StoredObject" - 403: - description: "Unauthorized" - 422: - description: "Invalid data" + /1.0/doc-store/stored-object/create: + post: + tags: + - storedobject + summary: Create a stored object + responses: + 200: + description: "OK" + content: + application/json: + schema: + $ref: "#/components/schemas/StoredObject" + 403: + description: "Unauthorized" + 422: + description: "Invalid data" + + /1.0/doc-store/async-upload/temp_url/{uuid}/generate/post: + get: + tags: + - storedobject + summary: Get a signed route to post stored object + parameters: + - in: path + name: uuid + required: true + allowEmptyValue: false + description: The UUID of the storedObject + schema: + type: string + format: uuid + responses: + 200: + description: "OK" + content: + application/json: + schema: + type: object + 403: + description: "Unauthorized" + 404: + description: "Not found" + /1.0/doc-store/async-upload/temp_url/{uuid}/generate/{method}: + get: + tags: + - storedobject + summary: Get a signed route to get a stored object + parameters: + - in: path + name: uuid + required: true + allowEmptyValue: false + description: The UUID of the storedObjeect + schema: + type: string + format: uuid + - in: path + name: method + required: true + allowEmptyValue: false + description: the method of the signed url (get or head) + schema: + type: string + enum: [get, head] + - in: query + name: version + required: false + allowEmptyValue: false + description: the version's filename of the stored object + schema: + type: string + minLength: 2 + responses: + 200: + description: "OK" + content: + application/json: + schema: + type: object + 403: + description: "Unauthorized" + 404: + description: "Not found" + From b6edbb3eedbb369cb6332ff9384077a81a3fdd2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 28 Aug 2024 23:19:24 +0200 Subject: [PATCH 17/25] Refactor StoredObject normalization handling Deprecate and remove specific context constants from StoredObjectNormalizer. Update object properties for better clarity and add permissions handling. Introduce related tests and adjust other files relying on the old context constants. --- .../Controller/AsyncUploadController.php | 2 +- .../StoredObjectDataTransformer.php | 7 +- .../Resources/public/types.ts | 40 +++++--- .../Normalizer/StoredObjectNormalizer.php | 29 ++++-- .../Normalizer/StoredObjectNormalizerTest.php | 98 +++++++++++++++++++ .../StoredObjectVersionNormalizerTest.php | 61 ++++++++++++ .../AccompanyingCourseWorkController.php | 3 +- 7 files changed, 211 insertions(+), 29 deletions(-) create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectNormalizerTest.php create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectVersionNormalizerTest.php diff --git a/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php b/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php index 879230cd9..52e0882a1 100644 --- a/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php +++ b/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php @@ -78,7 +78,7 @@ final readonly class AsyncUploadController if ($request->query->has('version')) { $filename = $request->query->get('version'); - $storedObjectVersion = $storedObject->getVersions()->findFirst(fn(int $index, StoredObjectVersion $version): bool => $version->getFilename() === $filename); + $storedObjectVersion = $storedObject->getVersions()->findFirst(fn (int $index, StoredObjectVersion $version): bool => $version->getFilename() === $filename); if (null === $storedObjectVersion) { // we are here in the case where the version is not stored into the database diff --git a/src/Bundle/ChillDocStoreBundle/Form/DataTransformer/StoredObjectDataTransformer.php b/src/Bundle/ChillDocStoreBundle/Form/DataTransformer/StoredObjectDataTransformer.php index e06d8d7cc..b5c7c3930 100644 --- a/src/Bundle/ChillDocStoreBundle/Form/DataTransformer/StoredObjectDataTransformer.php +++ b/src/Bundle/ChillDocStoreBundle/Form/DataTransformer/StoredObjectDataTransformer.php @@ -12,7 +12,6 @@ declare(strict_types=1); namespace Chill\DocStoreBundle\Form\DataTransformer; use Chill\DocStoreBundle\Entity\StoredObject; -use Chill\DocStoreBundle\Serializer\Normalizer\StoredObjectNormalizer; use Symfony\Component\Form\DataTransformerInterface; use Symfony\Component\Form\Exception\UnexpectedTypeException; use Symfony\Component\Serializer\SerializerInterface; @@ -30,11 +29,7 @@ class StoredObjectDataTransformer implements DataTransformerInterface } if ($value instanceof StoredObject) { - return $this->serializer->serialize($value, 'json', [ - 'groups' => [ - StoredObjectNormalizer::ADD_DAV_EDIT_LINK_CONTEXT, - ], - ]); + return $this->serializer->serialize($value, 'json'); } throw new UnexpectedTypeException($value, StoredObject::class); diff --git a/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts b/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts index 1d21feacd..235b375ce 100644 --- a/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts +++ b/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts @@ -1,28 +1,44 @@ -import {DateTime} from "../../../ChillMainBundle/Resources/public/types"; +import {DateTime, User} from "../../../ChillMainBundle/Resources/public/types"; export type StoredObjectStatus = "empty"|"ready"|"failure"|"pending"; export interface StoredObject { id: number, - - /** - * filename of the object in the object storage - */ - filename: string, - creationDate: DateTime, - datas: object, - iv: number[], - keyInfos: object, title: string, - type: string, uuid: string, + prefix: string, status: StoredObjectStatus, + currentVersion: null|StoredObjectVersion, + totalVersions: number, + datas: object, + /** @deprecated */ + creationDate: DateTime, + createdAt: DateTime|null, + createdBy: User|null, + _permissions: { + canEdit: boolean, + canSee: boolean, + }, _links?: { dav_link?: { href: string expiration: number }, - } + }, +} + +export interface StoredObjectVersion { + /** + * filename of the object in the object storage + */ + filename: string, + version: number, + id: number, + iv: number[], + keyInfos: object, + type: string, + createdAt: DateTime|null, + createdBy: User|null, } export interface StoredObjectCreated { diff --git a/src/Bundle/ChillDocStoreBundle/Serializer/Normalizer/StoredObjectNormalizer.php b/src/Bundle/ChillDocStoreBundle/Serializer/Normalizer/StoredObjectNormalizer.php index ebd7d3564..dbd9d72b8 100644 --- a/src/Bundle/ChillDocStoreBundle/Serializer/Normalizer/StoredObjectNormalizer.php +++ b/src/Bundle/ChillDocStoreBundle/Serializer/Normalizer/StoredObjectNormalizer.php @@ -16,6 +16,7 @@ use Chill\DocStoreBundle\Security\Authorization\StoredObjectRoleEnum; use Chill\DocStoreBundle\Security\Guard\JWTDavTokenProviderInterface; use Symfony\Component\Routing\Generator\UrlGeneratorInterface; use Symfony\Component\Security\Core\Security; +use Symfony\Component\Serializer\Normalizer\AbstractNormalizer; use Symfony\Component\Serializer\Normalizer\NormalizerAwareInterface; use Symfony\Component\Serializer\Normalizer\NormalizerAwareTrait; use Symfony\Component\Serializer\Normalizer\NormalizerInterface; @@ -28,8 +29,16 @@ use Symfony\Component\Serializer\Normalizer\NormalizerInterface; final class StoredObjectNormalizer implements NormalizerInterface, NormalizerAwareInterface { use NormalizerAwareTrait; - public const ADD_DAV_SEE_LINK_CONTEXT = 'dav-see-link-context'; - public const ADD_DAV_EDIT_LINK_CONTEXT = 'dav-edit-link-context'; + + /** + * @deprecated + */ + public const string ADD_DAV_SEE_LINK_CONTEXT = 'dav-see-link-context'; + + /** + * @deprecated + */ + public const string ADD_DAV_EDIT_LINK_CONTEXT = 'dav-edit-link-context'; public function __construct( private readonly JWTDavTokenProviderInterface $JWTDavTokenProvider, @@ -41,17 +50,16 @@ final class StoredObjectNormalizer implements NormalizerInterface, NormalizerAwa { /** @var StoredObject $object */ $datas = [ - 'datas' => $object->getDatas(), - 'filename' => $object->getFilename(), 'id' => $object->getId(), - 'iv' => $object->getIv(), - 'keyInfos' => $object->getKeyInfos(), + 'datas' => $object->getDatas(), + 'prefix' => $object->getPrefix(), 'title' => $object->getTitle(), - 'type' => $object->getType(), - 'uuid' => $object->getUuid(), + 'uuid' => $object->getUuid()->toString(), 'status' => $object->getStatus(), 'createdAt' => $this->normalizer->normalize($object->getCreatedAt(), $format, $context), 'createdBy' => $this->normalizer->normalize($object->getCreatedBy(), $format, $context), + 'currentVersion' => $this->normalizer->normalize($object->getCurrentVersion(), $format, [...$context, [AbstractNormalizer::GROUPS => 'read']]), + 'totalVersions' => $object->getVersions()->count(), ]; // deprecated property @@ -60,6 +68,11 @@ final class StoredObjectNormalizer implements NormalizerInterface, NormalizerAwa $canSee = $this->security->isGranted(StoredObjectRoleEnum::SEE->value, $object); $canEdit = $this->security->isGranted(StoredObjectRoleEnum::EDIT->value, $object); + $datas['_permissions'] = [ + 'canEdit' => $canEdit, + 'canSee' => $canSee, + ]; + if ($canSee || $canEdit) { $accessToken = $this->JWTDavTokenProvider->createToken( $object, diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectNormalizerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectNormalizerTest.php new file mode 100644 index 000000000..a2cc75541 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectNormalizerTest.php @@ -0,0 +1,98 @@ +setTitle('test'); + $reflection = new \ReflectionClass(StoredObject::class); + $idProperty = $reflection->getProperty('id'); + $idProperty->setValue($storedObject, 1); + + $jwtProvider = $this->createMock(JWTDavTokenProviderInterface::class); + $jwtProvider->expects($this->once())->method('createToken')->withAnyParameters()->willReturn('token'); + $jwtProvider->expects($this->once())->method('getTokenExpiration')->with('token')->willReturn($d = new \DateTimeImmutable()); + + $urlGenerator = $this->createMock(UrlGeneratorInterface::class); + $urlGenerator->expects($this->once())->method('generate') + ->with( + 'chill_docstore_dav_document_get', + [ + 'uuid' => $storedObject->getUuid(), + 'access_token' => 'token', + ], + UrlGeneratorInterface::ABSOLUTE_URL, + ) + ->willReturn($davLink = 'http://localhost/dav/token'); + + $security = $this->createMock(Security::class); + $security->expects($this->exactly(2))->method('isGranted') + ->with( + $this->logicalOr(StoredObjectRoleEnum::EDIT->value, StoredObjectRoleEnum::SEE->value), + $storedObject + ) + ->willReturn(true); + + $globalNormalizer = $this->createMock(NormalizerInterface::class); + $globalNormalizer->expects($this->exactly(3))->method('normalize') + ->withAnyParameters() + ->willReturnCallback(function (?object $object, string $format, array $context) { + if (null === $object) { + return null; + } + + return ['sub' => 'sub']; + }); + + $normalizer = new StoredObjectNormalizer($jwtProvider, $urlGenerator, $security); + $normalizer->setNormalizer($globalNormalizer); + + $actual = $normalizer->normalize($storedObject, 'json'); + + self::assertArrayHasKey('id', $actual); + self::assertEquals(1, $actual['id']); + self::assertArrayHasKey('title', $actual); + self::assertEquals('test', $actual['title']); + self::assertArrayHasKey('uuid', $actual); + self::assertArrayHasKey('prefix', $actual); + self::assertArrayHaskey('status', $actual); + self::assertArrayHasKey('currentVersion', $actual); + self::assertEquals(null, $actual['currentVersion']); + self::assertArrayHasKey('totalVersions', $actual); + self::assertEquals(0, $actual['totalVersions']); + self::assertArrayHasKey('datas', $actual); + self::assertArrayHasKey('createdAt', $actual); + self::assertArrayHasKey('createdBy', $actual); + self::assertArrayHasKey('_permissions', $actual); + self::assertEqualsCanonicalizing(['canEdit' => true, 'canSee' => true], $actual['_permissions']); + self::assertArrayHaskey('_links', $actual); + self::assertArrayHasKey('dav_link', $actual['_links']); + self::assertEqualsCanonicalizing(['href' => $davLink, 'expiration' => $d->getTimestamp()], $actual['_links']['dav_link']); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectVersionNormalizerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectVersionNormalizerTest.php new file mode 100644 index 000000000..c6bd87686 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectVersionNormalizerTest.php @@ -0,0 +1,61 @@ +normalizer = self::getContainer()->get(NormalizerInterface::class); + } + + public function testNormalize(): void + { + $storedObject = new StoredObject(); + $version = $storedObject->registerVersion( + iv: [1, 2, 3, 4], + keyInfos: ['someKey' => 'someKey'], + type: 'text/text', + ); + $reflection = new \ReflectionClass($version); + $idProperty = $reflection->getProperty('id'); + $idProperty->setValue($version, 1); + + $actual = $this->normalizer->normalize($version, 'json', ['groups' => ['read']]); + + self::assertEqualsCanonicalizing( + [ + 'id' => 1, + 'version' => 0, + 'filename' => $version->getFilename(), + 'iv' => [1, 2, 3, 4], + 'keyInfos' => ['someKey' => 'someKey'], + 'type' => 'text/text', + 'createdAt' => null, + 'createdBy' => null, + ], + $actual + ); + } +} diff --git a/src/Bundle/ChillPersonBundle/Controller/AccompanyingCourseWorkController.php b/src/Bundle/ChillPersonBundle/Controller/AccompanyingCourseWorkController.php index 0243d55b5..c43dbd128 100644 --- a/src/Bundle/ChillPersonBundle/Controller/AccompanyingCourseWorkController.php +++ b/src/Bundle/ChillPersonBundle/Controller/AccompanyingCourseWorkController.php @@ -11,7 +11,6 @@ declare(strict_types=1); namespace Chill\PersonBundle\Controller; -use Chill\DocStoreBundle\Serializer\Normalizer\StoredObjectNormalizer; use Chill\MainBundle\Pagination\PaginatorFactory; use Chill\MainBundle\Templating\Listing\FilterOrderHelper; use Chill\MainBundle\Templating\Listing\FilterOrderHelperFactoryInterface; @@ -116,7 +115,7 @@ final class AccompanyingCourseWorkController extends AbstractController { $this->denyAccessUnlessGranted(AccompanyingPeriodWorkVoter::UPDATE, $work); - $json = $this->serializer->normalize($work, 'json', ['groups' => ['read', StoredObjectNormalizer::ADD_DAV_EDIT_LINK_CONTEXT]]); + $json = $this->serializer->normalize($work, 'json', ['groups' => ['read']]); return $this->render('@ChillPerson/AccompanyingCourseWork/edit.html.twig', [ 'accompanyingCourse' => $work->getAccompanyingPeriod(), From 3d49c959e00335c3628c75e2e64078f569345fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 2 Sep 2024 16:24:23 +0200 Subject: [PATCH 18/25] Update DropFile to handle object versioning --- .gitlab-ci.yml | 2 +- .../TempUrlOpenstackGenerator.php | 8 +- .../Controller/AsyncUploadController.php | 10 +- .../Entity/StoredObject.php | 21 +-- .../Entity/StoredObjectVersion.php | 2 +- .../DataMapper/StoredObjectDataMapper.php | 11 +- .../StoredObjectDataTransformer.php | 4 +- .../Repository/StoredObjectRepository.php | 5 + .../StoredObjectRepositoryInterface.php | 2 + .../helper.ts => js/async-upload/uploader.ts} | 18 +- .../public/module/async_upload/index.ts | 14 +- .../Resources/public/types.ts | 25 ++- .../vuejs/DocumentActionButtonsGroup.vue | 55 ++++-- .../public/vuejs/DocumentSignature/App.vue | 8 +- .../public/vuejs/DropFileWidget/DropFile.vue | 52 +++--- .../vuejs/DropFileWidget/DropFileModal.vue | 69 +++++++ .../vuejs/DropFileWidget/DropFileWidget.vue | 14 +- .../StoredObjectButton/ConvertButton.vue | 4 +- .../StoredObjectButton/DesktopEditButton.vue | 3 + .../StoredObjectButton/DownloadButton.vue | 28 +-- .../StoredObjectButton/WopiEditButton.vue | 5 +- .../vuejs/StoredObjectButton/helpers.ts | 53 ++++-- .../vuejs/_components/AddAsyncUpload.vue | 174 ------------------ .../_components/AddAsyncUploadDownloader.vue | 45 ----- .../Normalizer/StoredObjectDenormalizer.php | 60 ++++-- .../Normalizer/StoredObjectNormalizer.php | 2 +- .../StoredObjectVersionNormalizer.php | 45 +++++ .../Service/StoredObjectManager.php | 35 +++- .../Service/StoredObjectManagerInterface.php | 6 + .../TempUrlOpenstackGeneratorTest.php | 80 +++++++- .../OpenstackObjectStore/file-to-upload.txt | 1 + .../Tests/Controller/WebdavControllerTest.php | 13 +- .../Tests/Entity/StoredObjectTest.php | 39 +--- .../Tests/Form/StoredObjectTypeTest.php | 62 ++++++- .../StoredObjectDenormalizerTest.php | 144 +++++++++++++++ .../StoredObjectVersionNormalizerTest.php | 27 ++- .../Constraints/AsyncFileExistsValidator.php | 63 ++----- .../WorkflowAddSignatureController.php | 64 +++++++ .../Controller/WorkflowController.php | 37 ---- .../components/FormEvaluation.vue | 41 ++--- .../vuejs/AccompanyingCourseWorkEdit/store.js | 14 +- tsconfig.json | 31 ++++ 42 files changed, 857 insertions(+), 539 deletions(-) rename src/Bundle/ChillDocStoreBundle/Resources/public/{vuejs/_components/helper.ts => js/async-upload/uploader.ts} (75%) create mode 100644 src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/DropFileWidget/DropFileModal.vue delete mode 100644 src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/_components/AddAsyncUpload.vue delete mode 100644 src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/_components/AddAsyncUploadDownloader.vue create mode 100644 src/Bundle/ChillDocStoreBundle/Serializer/Normalizer/StoredObjectVersionNormalizer.php create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/AsyncUpload/Driver/OpenstackObjectStore/file-to-upload.txt create mode 100644 src/Bundle/ChillDocStoreBundle/Tests/Serializer/Normalizer/StoredObjectDenormalizerTest.php create mode 100644 src/Bundle/ChillMainBundle/Controller/WorkflowAddSignatureController.php create mode 100644 tsconfig.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb5c8dd5d..e493e6556 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -122,7 +122,7 @@ unit_tests: - php tests/console chill:db:sync-views --env=test - php -d memory_limit=2G tests/console cache:clear --env=test - php -d memory_limit=3G tests/console doctrine:fixtures:load -n --env=test - - php -d memory_limit=4G bin/phpunit --colors=never --exclude-group dbIntensive + - php -d memory_limit=4G bin/phpunit --colors=never --exclude-group dbIntensive --exclude-group openstack-integration artifacts: expire_in: 1 day paths: diff --git a/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php index 3ba1f0dea..19a52f71d 100644 --- a/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php +++ b/src/Bundle/ChillDocStoreBundle/AsyncUpload/Driver/OpenstackObjectStore/TempUrlOpenstackGenerator.php @@ -182,21 +182,19 @@ final readonly class TempUrlOpenstackGenerator implements TempUrlGeneratorInterf return \hash_hmac('sha512', $body, $this->key, false); } - private function generateSignature($method, $url, \DateTimeImmutable $expires) + private function generateSignature(string $method, $url, \DateTimeImmutable $expires) { if ('POST' === $method) { return $this->generateSignaturePost($url, $expires); } $path = \parse_url((string) $url, PHP_URL_PATH); - $body = sprintf( "%s\n%s\n%s", - $method, + strtoupper($method), $expires->format('U'), $path - ) - ; + ); $this->logger->debug( 'generate signature GET', diff --git a/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php b/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php index 52e0882a1..cdbb94d29 100644 --- a/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php +++ b/src/Bundle/ChillDocStoreBundle/Controller/AsyncUploadController.php @@ -15,6 +15,7 @@ use Chill\DocStoreBundle\AsyncUpload\TempUrlGeneratorInterface; use Chill\DocStoreBundle\Entity\StoredObject; use Chill\DocStoreBundle\Entity\StoredObjectVersion; use Chill\DocStoreBundle\Security\Authorization\StoredObjectRoleEnum; +use Chill\MainBundle\Entity\User; use Psr\Log\LoggerInterface; use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\HttpFoundation\Request; @@ -99,8 +100,15 @@ final readonly class AsyncUploadController $request->query->has('expires_delay') ? $request->query->getInt('expires_delay') : null ); + $user = $this->security->getUser(); + $userId = match ($user instanceof User) { + true => $user->getId(), + false => $user->getUserIdentifier(), + }; + $this->chillLogger->notice('[Privacy Event] a request to see a document has been granted', [ - 'doc_uuid' => $storedObject->getUuid(), + 'doc_uuid' => $storedObject->getUuid()->toString(), + 'user_id' => $userId, ]); return new JsonResponse( diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php index 6ed52716d..dabcec8eb 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObject.php @@ -23,6 +23,7 @@ use Ramsey\Uuid\Uuid; use Ramsey\Uuid\UuidInterface; use Random\RandomException; use Symfony\Component\Serializer\Annotation as Serializer; +use Symfony\Component\Validator\Constraints as Assert; /** * Represent a document stored in an object store. @@ -37,7 +38,6 @@ use Symfony\Component\Serializer\Annotation as Serializer; */ #[ORM\Entity] #[ORM\Table('stored_object', schema: 'chill_doc')] -#[AsyncFileExists(message: 'The file is not stored properly')] class StoredObject implements Document, TrackCreationInterface { use TrackCreationTrait; @@ -91,7 +91,7 @@ class StoredObject implements Document, TrackCreationInterface /** * @var Collection */ - #[ORM\OneToMany(targetEntity: StoredObjectVersion::class, cascade: ['persist'], mappedBy: 'storedObject', orphanRemoval: true)] + #[ORM\OneToMany(mappedBy: 'storedObject', targetEntity: StoredObjectVersion::class, cascade: ['persist'], orphanRemoval: true)] private Collection $versions; /** @@ -127,6 +127,8 @@ class StoredObject implements Document, TrackCreationInterface return \DateTime::createFromImmutable($this->createdAt); } + #[AsyncFileExists(message: 'The file is not stored properly')] + #[Assert\NotNull(message: 'The store object version must be present')] public function getCurrentVersion(): ?StoredObjectVersion { $maxVersion = null; @@ -350,20 +352,7 @@ class StoredObject implements Document, TrackCreationInterface /** * @deprecated */ - public function saveHistory(): void - { - if ('' === $this->getFilename()) { - return; - } - - $this->datas['history'][] = [ - 'filename' => $this->getFilename(), - 'iv' => $this->getIv(), - 'key_infos' => $this->getKeyInfos(), - 'type' => $this->getType(), - 'before' => (new \DateTimeImmutable('now'))->getTimestamp(), - ]; - } + public function saveHistory(): void {} public static function generatePrefix(): string { diff --git a/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php b/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php index 84b10688f..8526cd9a9 100644 --- a/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php +++ b/src/Bundle/ChillDocStoreBundle/Entity/StoredObjectVersion.php @@ -86,7 +86,7 @@ class StoredObjectVersion implements TrackCreationInterface { try { $suffix = base_convert(bin2hex(random_bytes(8)), 16, 36); - } catch (RandomException $e) { + } catch (RandomException) { $suffix = uniqid(more_entropy: true); } diff --git a/src/Bundle/ChillDocStoreBundle/Form/DataMapper/StoredObjectDataMapper.php b/src/Bundle/ChillDocStoreBundle/Form/DataMapper/StoredObjectDataMapper.php index 37d28c7a4..264f52365 100644 --- a/src/Bundle/ChillDocStoreBundle/Form/DataMapper/StoredObjectDataMapper.php +++ b/src/Bundle/ChillDocStoreBundle/Form/DataMapper/StoredObjectDataMapper.php @@ -55,15 +55,8 @@ class StoredObjectDataMapper implements DataMapperInterface return; } - /** @var StoredObject $viewData */ - if ($viewData->getFilename() !== $forms['stored_object']->getData()['filename']) { - $viewData->registerVersion( - $forms['stored_object']->getData()['iv'], - $forms['stored_object']->getData()['keyInfos'], - $forms['stored_object']->getData()['type'], - $forms['stored_object']->getData()['filename'], - ); - } + /* @var StoredObject $viewData */ + $viewData = $forms['stored_object']->getData(); if (array_key_exists('title', $forms)) { $viewData->setTitle($forms['title']->getData()); diff --git a/src/Bundle/ChillDocStoreBundle/Form/DataTransformer/StoredObjectDataTransformer.php b/src/Bundle/ChillDocStoreBundle/Form/DataTransformer/StoredObjectDataTransformer.php index b5c7c3930..8df3e3eb9 100644 --- a/src/Bundle/ChillDocStoreBundle/Form/DataTransformer/StoredObjectDataTransformer.php +++ b/src/Bundle/ChillDocStoreBundle/Form/DataTransformer/StoredObjectDataTransformer.php @@ -19,7 +19,7 @@ use Symfony\Component\Serializer\SerializerInterface; class StoredObjectDataTransformer implements DataTransformerInterface { public function __construct( - private readonly SerializerInterface $serializer + private readonly SerializerInterface $serializer, ) {} public function transform(mixed $value): mixed @@ -41,6 +41,6 @@ class StoredObjectDataTransformer implements DataTransformerInterface return null; } - return json_decode((string) $value, true, 10, JSON_THROW_ON_ERROR); + return $this->serializer->deserialize($value, StoredObject::class, 'json'); } } diff --git a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepository.php b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepository.php index 2f39f2021..d48792bc9 100644 --- a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepository.php +++ b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepository.php @@ -64,6 +64,11 @@ final readonly class StoredObjectRepository implements StoredObjectRepositoryInt return $qb->getQuery()->toIterable(hydrationMode: Query::HYDRATE_OBJECT); } + public function findOneByUUID(string $uuid): ?StoredObject + { + return $this->repository->findOneBy(['uuid' => $uuid]); + } + public function getClassName(): string { return StoredObject::class; diff --git a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepositoryInterface.php b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepositoryInterface.php index 45dcfcf94..877847677 100644 --- a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepositoryInterface.php +++ b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectRepositoryInterface.php @@ -23,4 +23,6 @@ interface StoredObjectRepositoryInterface extends ObjectRepository * @return iterable */ public function findByExpired(\DateTimeImmutable $expiredAtDate): iterable; + + public function findOneByUUID(string $uuid): ?StoredObject; } diff --git a/src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/_components/helper.ts b/src/Bundle/ChillDocStoreBundle/Resources/public/js/async-upload/uploader.ts similarity index 75% rename from src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/_components/helper.ts rename to src/Bundle/ChillDocStoreBundle/Resources/public/js/async-upload/uploader.ts index 6d07e769d..592bd8111 100644 --- a/src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/_components/helper.ts +++ b/src/Bundle/ChillDocStoreBundle/Resources/public/js/async-upload/uploader.ts @@ -1,5 +1,5 @@ import {makeFetch} from "../../../../../ChillMainBundle/Resources/public/lib/api/apiMethods"; -import {PostStoreObjectSignature} from "../../types"; +import {PostStoreObjectSignature, StoredObject} from "../../types"; const algo = 'AES-CBC'; @@ -21,11 +21,22 @@ const createFilename = (): string => { return text; }; -export const uploadFile = async (uploadFile: ArrayBuffer): Promise => { +/** + * Fetches a new stored object from the server. + * + * @async + * @function fetchNewStoredObject + * @returns {Promise} A Promise that resolves to the newly created StoredObject. + */ +export const fetchNewStoredObject = async (): Promise => { + return makeFetch("POST", '/api/1.0/doc-store/stored-object/create', null); +} + +export const uploadVersion = async (uploadFile: ArrayBuffer, storedObject: StoredObject): Promise => { const params = new URLSearchParams(); params.append('expires_delay', "180"); params.append('submit_delay', "180"); - const asyncData: PostStoreObjectSignature = await makeFetch("GET", URL_POST + "?" + params.toString()); + const asyncData: PostStoreObjectSignature = await makeFetch("GET", `/api/1.0/doc-store/async-upload/temp_url/${storedObject.uuid}/generate/post` + "?" + params.toString()); const suffix = createFilename(); const filename = asyncData.prefix + suffix; const formData = new FormData(); @@ -50,7 +61,6 @@ export const uploadFile = async (uploadFile: ArrayBuffer): Promise => { } export const encryptFile = async (originalFile: ArrayBuffer): Promise<[ArrayBuffer, Uint8Array, JsonWebKey]> => { - console.log('encrypt', originalFile); const iv = crypto.getRandomValues(new Uint8Array(16)); const key = await window.crypto.subtle.generateKey(keyDefinition, true, [ "encrypt", "decrypt" ]); const exportedKey = await window.crypto.subtle.exportKey('jwk', key); diff --git a/src/Bundle/ChillDocStoreBundle/Resources/public/module/async_upload/index.ts b/src/Bundle/ChillDocStoreBundle/Resources/public/module/async_upload/index.ts index b7df11323..50849d635 100644 --- a/src/Bundle/ChillDocStoreBundle/Resources/public/module/async_upload/index.ts +++ b/src/Bundle/ChillDocStoreBundle/Resources/public/module/async_upload/index.ts @@ -1,7 +1,7 @@ import {CollectionEventPayload} from "../../../../../ChillMainBundle/Resources/public/module/collection"; import {createApp} from "vue"; import DropFileWidget from "../../vuejs/DropFileWidget/DropFileWidget.vue" -import {StoredObject, StoredObjectCreated} from "../../types"; +import {StoredObject, StoredObjectVersion} from "../../types"; import {_createI18n} from "../../../../../ChillMainBundle/Resources/public/vuejs/_js/i18n"; const i18n = _createI18n({}); @@ -30,15 +30,17 @@ const startApp = (divElement: HTMLDivElement, collectionEntry: null|HTMLLIElemen DropFileWidget, }, methods: { - addDocument: function(object: StoredObjectCreated): void { - console.log('object added', object); - this.$data.existingDoc = object; - input_stored_object.value = JSON.stringify(object); + addDocument: function({stored_object, stored_object_version}: {stored_object: StoredObject, stored_object_version: StoredObjectVersion}): void { + console.log('object added', stored_object); + console.log('version added', stored_object_version); + this.$data.existingDoc = stored_object; + this.$data.existingDoc.currentVersion = stored_object_version; + input_stored_object.value = JSON.stringify(this.$data.existingDoc); }, removeDocument: function(object: StoredObject): void { console.log('catch remove document', object); input_stored_object.value = ""; - this.$data.existingDoc = null; + this.$data.existingDoc = undefined; console.log('collectionEntry', collectionEntry); if (null !== collectionEntry) { diff --git a/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts b/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts index 235b375ce..840421991 100644 --- a/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts +++ b/src/Bundle/ChillDocStoreBundle/Resources/public/types.ts @@ -4,11 +4,11 @@ export type StoredObjectStatus = "empty"|"ready"|"failure"|"pending"; export interface StoredObject { id: number, - title: string, + title: string|null, uuid: string, prefix: string, status: StoredObjectStatus, - currentVersion: null|StoredObjectVersion, + currentVersion: null|StoredObjectVersionCreated|StoredObjectVersionPersisted, totalVersions: number, datas: object, /** @deprecated */ @@ -32,21 +32,20 @@ export interface StoredObjectVersion { * filename of the object in the object storage */ filename: string, - version: number, - id: number, iv: number[], - keyInfos: object, + keyInfos: JsonWebKey, type: string, - createdAt: DateTime|null, - createdBy: User|null, } -export interface StoredObjectCreated { - status: "stored_object_created", - filename: string, - iv: Uint8Array, - keyInfos: object, - type: string, +export interface StoredObjectVersionCreated extends StoredObjectVersion { + persisted: false, +} + +export interface StoredObjectVersionPersisted extends StoredObjectVersionCreated { + version: number, + id: number, + createdAt: DateTime|null, + createdBy: User|null, } export interface StoredObjectStatusChange { diff --git a/src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/DocumentActionButtonsGroup.vue b/src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/DocumentActionButtonsGroup.vue index b4c53eacd..66546f2a5 100644 --- a/src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/DocumentActionButtonsGroup.vue +++ b/src/Bundle/ChillDocStoreBundle/Resources/public/vuejs/DocumentActionButtonsGroup.vue @@ -1,20 +1,20 @@