Add functionality to delete old versions of documents

This commit introduces a feature that automatically deletes old versions of StoredObjects in the Chill application. A cron job, "RemoveOldVersionCronJob", has been implemented to delete versions older than 90 days. A message handler, "RemoveOldVersionMessageHandler", has been added to handle deletion requests. Furthermore, unit tests for the new functionality have been provided.

src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php

@@ -0,0 +1,92 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\Repository;
+
+use Chill\DocStoreBundle\Entity\StoredObjectVersion;
+use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\Types\Types;
+use Doctrine\ORM\EntityManagerInterface;
+use Doctrine\ORM\EntityRepository;
+use Doctrine\Persistence\ObjectRepository;
+
+/**
+ * @implements ObjectRepository<StoredObjectVersion>
+ */
+class StoredObjectVersionRepository implements ObjectRepository
+{
+    private EntityRepository $repository;
+
+    private Connection $connection;
+
+    public function __construct(EntityManagerInterface $entityManager)
+    {
+        $this->repository = $entityManager->getRepository(StoredObjectVersion::class);
+        $this->connection = $entityManager->getConnection();
+    }
+
+    public function find($id): ?StoredObjectVersion
+    {
+        return $this->repository->find($id);
+    }
+
+    public function findAll(): array
+    {
+        return $this->repository->findAll();
+    }
+
+    public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): array
+    {
+        return $this->repository->findBy($criteria, $orderBy, $limit, $offset);
+    }
+
+    public function findOneBy(array $criteria): ?StoredObjectVersion
+    {
+        return $this->repository->findOneBy($criteria);
+    }
+
+    /**
+     * Finds the IDs of versions older than a given date and that are not the last version.
+     *
+     * Those version are good candidates for a deletion.
+     *
+     * @param \DateTimeImmutable $beforeDate the date to compare versions against
+     *
+     * @return iterable returns an iterable with the IDs of the versions
+     */
+    public function findIdsByVersionsOlderThanDateAndNotLastVersion(\DateTimeImmutable $beforeDate): iterable
+    {
+        $results = $this->connection->executeQuery(
+            self::QUERY_FIND_IDS_BY_VERSIONS_OLDER_THAN_DATE_AND_NOT_LAST_VERSION,
+            [$beforeDate],
+            [Types::DATETIME_IMMUTABLE]
+        );
+
+        foreach ($results->iterateAssociative() as $row) {
+            yield $row['sov_id'];
+        }
+    }
+
+    private const QUERY_FIND_IDS_BY_VERSIONS_OLDER_THAN_DATE_AND_NOT_LAST_VERSION = <<<'SQL'
+        SELECT
+            sov.id AS sov_id
+        FROM chill_doc.stored_object_version sov
+        WHERE
+            sov.createdat < ?::timestamp
+            AND
+            sov.version < (SELECT MAX(sub_sov.version) FROM chill_doc.stored_object_version sub_sov WHERE sub_sov.stored_object_id = sov.stored_object_id)
+        SQL;
+
+    public function getClassName(): string
+    {
+        return StoredObjectVersion::class;
+    }
+}

src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionCronJob.php

@@ -0,0 +1,60 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\Service\StoredObjectCleaner;
+
+use Chill\DocStoreBundle\Repository\StoredObjectVersionRepository;
+use Chill\MainBundle\Cron\CronJobInterface;
+use Chill\MainBundle\Entity\CronJobExecution;
+use Symfony\Component\Clock\ClockInterface;
+use Symfony\Component\Messenger\MessageBusInterface;
+
+final readonly class RemoveOldVersionCronJob implements CronJobInterface
+{
+    public const KEY = 'remove-old-stored-object-version';
+
+    private const LAST_DELETED_KEY = 'last-deleted-stored-object-version-id';
+
+    public const KEEP_INTERVAL = 'P90D';
+
+    public function __construct(
+        private ClockInterface $clock,
+        private MessageBusInterface $messageBus,
+        private StoredObjectVersionRepository $storedObjectVersionRepository,
+    ) {}
+
+    public function canRun(?CronJobExecution $cronJobExecution): bool
+    {
+        if (null === $cronJobExecution) {
+            return true;
+        }
+
+        return $this->clock->now() >= $cronJobExecution->getLastEnd()->add(new \DateInterval('P1D'));
+    }
+
+    public function getKey(): string
+    {
+        return self::KEY;
+    }
+
+    public function run(array $lastExecutionData): ?array
+    {
+        $deleteBeforeDate = $this->clock->now()->sub(new \DateInterval(self::KEEP_INTERVAL));
+        $maxDeleted = $lastExecutionData[self::LAST_DELETED_KEY] ?? 0;
+
+        foreach ($this->storedObjectVersionRepository->findIdsByVersionsOlderThanDateAndNotLastVersion($deleteBeforeDate) as $id) {
+            $this->messageBus->dispatch(new RemoveOldVersionMessage($id));
+            $maxDeleted = max($maxDeleted, $id);
+        }
+
+        return [self::LAST_DELETED_KEY => $maxDeleted];
+    }
+}

src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessage.php

@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\Service\StoredObjectCleaner;
+
+final readonly class RemoveOldVersionMessage
+{
+    public function __construct(
+        public int $storedObjectVersionId
+    ) {}
+}

src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessageHandler.php

@@ -0,0 +1,54 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\Service\StoredObjectCleaner;
+
+use Chill\DocStoreBundle\Exception\StoredObjectManagerException;
+use Chill\DocStoreBundle\Repository\StoredObjectVersionRepository;
+use Chill\DocStoreBundle\Service\StoredObjectManagerInterface;
+use Doctrine\ORM\EntityManagerInterface;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Messenger\Handler\MessageHandlerInterface;
+
+final readonly class RemoveOldVersionMessageHandler implements MessageHandlerInterface
+{
+    private const LOG_PREFIX = '[RemoveOldVersionMessageHandler] ';
+
+    public function __construct(
+        private StoredObjectVersionRepository $storedObjectVersionRepository,
+        private LoggerInterface $logger,
+        private EntityManagerInterface $entityManager,
+        private StoredObjectManagerInterface $storedObjectManager,
+    ) {}
+
+    /**
+     * @throws StoredObjectManagerException
+     */
+    public function __invoke(RemoveOldVersionMessage $message): void
+    {
+        $this->logger->info(self::LOG_PREFIX.'Received one message', ['storedObjectVersionId' => $message->storedObjectVersionId]);
+
+        $storedObjectVersion = $this->storedObjectVersionRepository->find($message->storedObjectVersionId);
+
+        if (null === $storedObjectVersion) {
+            $this->logger->error(self::LOG_PREFIX.'StoredObjectVersion not found in database', ['storedObjectVersionId' => $message->storedObjectVersionId]);
+            throw new \RuntimeException('StoredObjectVersion not found with id '.$message->storedObjectVersionId);
+        }
+
+        $this->storedObjectManager->delete($storedObjectVersion);
+
+        $this->entityManager->remove($storedObjectVersion);
+        $this->entityManager->flush();
+
+        // clear the entity manager for future usage
+        $this->entityManager->clear();
+    }
+}

src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php

@@ -217,6 +217,23 @@ final class StoredObjectManager implements StoredObjectManagerInterface
         return $version;
     }
 
+    /**
+     * @throws StoredObjectManagerException
+     */
+    public function delete(StoredObjectVersion $storedObjectVersion): void
+    {
+        $signedUrl = $this->tempUrlGenerator->generate('DELETE', $storedObjectVersion->getFilename());
+
+        try {
+            $response = $this->client->request('DELETE', $signedUrl->url);
+            if (Response::HTTP_NO_CONTENT !== $response->getStatusCode()) {
+                throw StoredObjectManagerException::invalidStatusCode($response->getStatusCode());
+            }
+        } catch (TransportExceptionInterface $exception) {
+            throw StoredObjectManagerException::errorDuringHttpRequest($exception);
+        }
+    }
+
     public function clearCache(): void
     {
         $this->inMemory = [];

src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php

@@ -51,6 +51,11 @@ interface StoredObjectManagerInterface
      */
     public function write(StoredObject $document, string $clearContent, ?string $contentType = null): StoredObjectVersion;
 
+    /**
+     * @throws StoredObjectManagerException
+     */
+    public function delete(StoredObjectVersion $storedObjectVersion): void;
+
     /**
      * return or compute the etag for the document.
      *

src/Bundle/ChillDocStoreBundle/Tests/Service/Repository/StoredObjectVersionRepositoryTest.php

@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Chill\DocStoreBundle\Tests\Repository;
+
+use Chill\DocStoreBundle\Repository\StoredObjectVersionRepository;
+use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
+
+/**
+ * @internal
+ *
+ * @coversNothing
+ */
+class StoredObjectVersionRepositoryTest extends KernelTestCase
+{
+    private EntityManagerInterface $entityManager;
+
+    protected function setUp(): void
+    {
+        self::bootKernel();
+        $this->entityManager = self::getContainer()->get(EntityManagerInterface::class);
+    }
+
+    public function testFindIdsByVersionsOlderThanDateAndNotLastVersion(): void
+    {
+        $repository = new StoredObjectVersionRepository($this->entityManager);
+
+        // get old version, to get a chance to get one
+        $actual = $repository->findIdsByVersionsOlderThanDateAndNotLastVersion(new \DateTimeImmutable('1970-01-01'));
+
+        self::assertIsIterable($actual);
+        self::assertContainsOnly('int', $actual);
+    }
+}

src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php

@@ -0,0 +1,104 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Tests\Service\StoredObjectCleaner;
+
+use Chill\DocStoreBundle\Repository\StoredObjectVersionRepository;
+use Chill\DocStoreBundle\Service\StoredObjectCleaner\RemoveOldVersionCronJob;
+use Chill\DocStoreBundle\Service\StoredObjectCleaner\RemoveOldVersionMessage;
+use Chill\MainBundle\Entity\CronJobExecution;
+use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
+use Symfony\Component\Clock\MockClock;
+use Symfony\Component\Messenger\Envelope;
+use Symfony\Component\Messenger\MessageBusInterface;
+
+/**
+ * @internal
+ *
+ * @coversNothing
+ */
+class RemoveOldVersionCronJobTest extends KernelTestCase
+{
+    /**
+     * @dataProvider buildTestCanRunData
+     */
+    public function testCanRun(?CronJobExecution $cronJobExecution, bool $expected): void
+    {
+        $repository = $this->createMock(StoredObjectVersionRepository::class);
+        $clock = new MockClock(new \DateTimeImmutable('2024-01-01 00:00:00', new \DateTimeZone('+00:00')));
+
+        $cronJob = new RemoveOldVersionCronJob($clock, $this->buildMessageBus(), $repository);
+
+        self::assertEquals($expected, $cronJob->canRun($cronJobExecution));
+    }
+
+    public function testRun(): void
+    {
+        // we create a clock in the future. This led us a chance to having stored object to delete
+        $clock = new MockClock(new \DateTimeImmutable('2024-01-01 00:00:00', new \DateTimeZone('+00:00')));
+        $repository = $this->createMock(StoredObjectVersionRepository::class);
+        $repository->expects($this->once())
+            ->method('findIdsByVersionsOlderThanDateAndNotLastVersion')
+            ->with(new \DateTime('2023-10-03 00:00:00', new \DateTimeZone('+00:00')))
+            ->willReturnCallback(function ($arg) {
+                yield 1;
+                yield 3;
+                yield 2;
+            })
+        ;
+
+        $cronJob = new RemoveOldVersionCronJob($clock, $this->buildMessageBus(true), $repository);
+
+        $results = $cronJob->run([]);
+
+        self::assertArrayHasKey('last-deleted-stored-object-version-id', $results);
+        self::assertIsInt($results['last-deleted-stored-object-version-id']);
+    }
+
+    public static function buildTestCanRunData(): iterable
+    {
+        yield [
+            (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-31 00:00:00', new \DateTimeZone('+00:00'))),
+            true,
+        ];
+
+        yield [
+            (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-30 23:59:59', new \DateTimeZone('+00:00'))),
+            true,
+        ];
+
+        yield [
+            (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-31 00:00:01', new \DateTimeZone('+00:00'))),
+            false,
+        ];
+
+        yield [
+            null,
+            true,
+        ];
+    }
+
+    private function buildMessageBus(bool $expectDistpatchAtLeastOnce = false): MessageBusInterface
+    {
+        $messageBus = $this->createMock(MessageBusInterface::class);
+
+        $methodDispatch = match ($expectDistpatchAtLeastOnce) {
+            true => $messageBus->expects($this->atLeastOnce())->method('dispatch')->with($this->isInstanceOf(RemoveOldVersionMessage::class)),
+            false => $messageBus->method('dispatch'),
+        };
+
+        $methodDispatch->willReturnCallback(function (RemoveOldVersionMessage $message) {
+            return new Envelope($message);
+        });
+
+        return $messageBus;
+    }
+}

src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php

@@ -0,0 +1,51 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+namespace Tests\Service\StoredObjectCleaner;
+
+use Chill\DocStoreBundle\Entity\StoredObject;
+use Chill\DocStoreBundle\Repository\StoredObjectVersionRepository;
+use Chill\DocStoreBundle\Service\StoredObjectCleaner\RemoveOldVersionMessage;
+use Chill\DocStoreBundle\Service\StoredObjectCleaner\RemoveOldVersionMessageHandler;
+use Chill\DocStoreBundle\Service\StoredObjectManagerInterface;
+use Doctrine\ORM\EntityManagerInterface;
+use PHPUnit\Framework\TestCase;
+use Psr\Log\NullLogger;
+
+/**
+ * @internal
+ *
+ * @coversNothing
+ */
+class RemoveOldVersionMessageHandlerTest extends TestCase
+{
+    public function testInvoke(): void
+    {
+        $object = new StoredObject();
+        $version = $object->registerVersion();
+        $storedObjectVersionRepository = $this->createMock(StoredObjectVersionRepository::class);
+        $storedObjectVersionRepository->expects($this->once())->method('find')
+            ->with($this->identicalTo(1))
+            ->willReturn($version);
+
+        $entityManager = $this->createMock(EntityManagerInterface::class);
+        $entityManager->expects($this->once())->method('remove')->with($this->identicalTo($version));
+        $entityManager->expects($this->once())->method('flush');
+        $entityManager->expects($this->once())->method('clear');
+
+        $storedObjectManager = $this->createMock(StoredObjectManagerInterface::class);
+        $storedObjectManager->expects($this->once())->method('delete')->with($this->identicalTo($version));
+
+        $handler = new RemoveOldVersionMessageHandler($storedObjectVersionRepository, new NullLogger(), $entityManager, $storedObjectManager);
+
+        $handler(new RemoveOldVersionMessage(1));
+    }
+}

src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php

@@ -198,6 +198,37 @@ final class StoredObjectManagerTest extends TestCase
         self::assertSame($storedObject->getCurrentVersion(), $newVersion);
     }
 
+    public function testDelete(): void
+    {
+        $storedObject = new StoredObject();
+        $version = $storedObject->registerVersion(filename: 'object_name');
+
+        $httpClient = new MockHttpClient(function ($method, $url, $options) {
+            self::assertEquals('DELETE', $method);
+            self::assertEquals('https://example.com/object_name', $url);
+
+            return new MockResponse('', [
+                'http_code' => 204,
+            ]);
+        });
+
+        $tempUrlGenerator = $this->createMock(TempUrlGeneratorInterface::class);
+        $tempUrlGenerator
+            ->expects($this->once())
+            ->method('generate')
+            ->with($this->identicalTo('DELETE'), $this->identicalTo('object_name'))
+            ->willReturnCallback(function (string $method, string $objectName) {
+                return new SignedUrl(
+                    $method,
+                    'https://example.com/'.$objectName,
+                    new \DateTimeImmutable('1 hours')
+                );
+            });
+
+        $storedObjectManager = new StoredObjectManager($httpClient, $tempUrlGenerator);
+        $storedObjectManager->delete($version);
+    }
+
     public function testWriteWithDeleteAt()
     {
         $storedObject = new StoredObject();