diff --git a/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php new file mode 100644 index 000000000..047b024a9 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Repository/StoredObjectVersionRepository.php @@ -0,0 +1,92 @@ + + */ +class StoredObjectVersionRepository implements ObjectRepository +{ + private EntityRepository $repository; + + private Connection $connection; + + public function __construct(EntityManagerInterface $entityManager) + { + $this->repository = $entityManager->getRepository(StoredObjectVersion::class); + $this->connection = $entityManager->getConnection(); + } + + public function find($id): ?StoredObjectVersion + { + return $this->repository->find($id); + } + + public function findAll(): array + { + return $this->repository->findAll(); + } + + public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): array + { + return $this->repository->findBy($criteria, $orderBy, $limit, $offset); + } + + public function findOneBy(array $criteria): ?StoredObjectVersion + { + return $this->repository->findOneBy($criteria); + } + + /** + * Finds the IDs of versions older than a given date and that are not the last version. + * + * Those version are good candidates for a deletion. + * + * @param \DateTimeImmutable $beforeDate the date to compare versions against + * + * @return iterable returns an iterable with the IDs of the versions + */ + public function findIdsByVersionsOlderThanDateAndNotLastVersion(\DateTimeImmutable $beforeDate): iterable + { + $results = $this->connection->executeQuery( + self::QUERY_FIND_IDS_BY_VERSIONS_OLDER_THAN_DATE_AND_NOT_LAST_VERSION, + [$beforeDate], + [Types::DATETIME_IMMUTABLE] + ); + + foreach ($results->iterateAssociative() as $row) { + yield $row['sov_id']; + } + } + + private const QUERY_FIND_IDS_BY_VERSIONS_OLDER_THAN_DATE_AND_NOT_LAST_VERSION = <<<'SQL' + SELECT + sov.id AS sov_id + FROM chill_doc.stored_object_version sov + WHERE + sov.createdat < ?::timestamp + AND + sov.version < (SELECT MAX(sub_sov.version) FROM chill_doc.stored_object_version sub_sov WHERE sub_sov.stored_object_id = sov.stored_object_id) + SQL; + + public function getClassName(): string + { + return StoredObjectVersion::class; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionCronJob.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionCronJob.php new file mode 100644 index 000000000..d190a4e45 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionCronJob.php @@ -0,0 +1,60 @@ +clock->now() >= $cronJobExecution->getLastEnd()->add(new \DateInterval('P1D')); + } + + public function getKey(): string + { + return self::KEY; + } + + public function run(array $lastExecutionData): ?array + { + $deleteBeforeDate = $this->clock->now()->sub(new \DateInterval(self::KEEP_INTERVAL)); + $maxDeleted = $lastExecutionData[self::LAST_DELETED_KEY] ?? 0; + + foreach ($this->storedObjectVersionRepository->findIdsByVersionsOlderThanDateAndNotLastVersion($deleteBeforeDate) as $id) { + $this->messageBus->dispatch(new RemoveOldVersionMessage($id)); + $maxDeleted = max($maxDeleted, $id); + } + + return [self::LAST_DELETED_KEY => $maxDeleted]; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessage.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessage.php new file mode 100644 index 000000000..a151e714c --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectCleaner/RemoveOldVersionMessage.php @@ -0,0 +1,19 @@ +logger->info(self::LOG_PREFIX.'Received one message', ['storedObjectVersionId' => $message->storedObjectVersionId]); + + $storedObjectVersion = $this->storedObjectVersionRepository->find($message->storedObjectVersionId); + + if (null === $storedObjectVersion) { + $this->logger->error(self::LOG_PREFIX.'StoredObjectVersion not found in database', ['storedObjectVersionId' => $message->storedObjectVersionId]); + throw new \RuntimeException('StoredObjectVersion not found with id '.$message->storedObjectVersionId); + } + + $this->storedObjectManager->delete($storedObjectVersion); + + $this->entityManager->remove($storedObjectVersion); + $this->entityManager->flush(); + + // clear the entity manager for future usage + $this->entityManager->clear(); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php index 2266bb0e2..51f32b98a 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManager.php @@ -217,6 +217,23 @@ final class StoredObjectManager implements StoredObjectManagerInterface return $version; } + /** + * @throws StoredObjectManagerException + */ + public function delete(StoredObjectVersion $storedObjectVersion): void + { + $signedUrl = $this->tempUrlGenerator->generate('DELETE', $storedObjectVersion->getFilename()); + + try { + $response = $this->client->request('DELETE', $signedUrl->url); + if (Response::HTTP_NO_CONTENT !== $response->getStatusCode()) { + throw StoredObjectManagerException::invalidStatusCode($response->getStatusCode()); + } + } catch (TransportExceptionInterface $exception) { + throw StoredObjectManagerException::errorDuringHttpRequest($exception); + } + } + public function clearCache(): void { $this->inMemory = []; diff --git a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php index 4d2f45c33..7d4711b06 100644 --- a/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php +++ b/src/Bundle/ChillDocStoreBundle/Service/StoredObjectManagerInterface.php @@ -51,6 +51,11 @@ interface StoredObjectManagerInterface */ public function write(StoredObject $document, string $clearContent, ?string $contentType = null): StoredObjectVersion; + /** + * @throws StoredObjectManagerException + */ + public function delete(StoredObjectVersion $storedObjectVersion): void; + /** * return or compute the etag for the document. * diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/Repository/StoredObjectVersionRepositoryTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/Repository/StoredObjectVersionRepositoryTest.php new file mode 100644 index 000000000..ace122bea --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/Repository/StoredObjectVersionRepositoryTest.php @@ -0,0 +1,43 @@ +entityManager = self::getContainer()->get(EntityManagerInterface::class); + } + + public function testFindIdsByVersionsOlderThanDateAndNotLastVersion(): void + { + $repository = new StoredObjectVersionRepository($this->entityManager); + + // get old version, to get a chance to get one + $actual = $repository->findIdsByVersionsOlderThanDateAndNotLastVersion(new \DateTimeImmutable('1970-01-01')); + + self::assertIsIterable($actual); + self::assertContainsOnly('int', $actual); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php new file mode 100644 index 000000000..f27727d20 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionCronJobTest.php @@ -0,0 +1,104 @@ +createMock(StoredObjectVersionRepository::class); + $clock = new MockClock(new \DateTimeImmutable('2024-01-01 00:00:00', new \DateTimeZone('+00:00'))); + + $cronJob = new RemoveOldVersionCronJob($clock, $this->buildMessageBus(), $repository); + + self::assertEquals($expected, $cronJob->canRun($cronJobExecution)); + } + + public function testRun(): void + { + // we create a clock in the future. This led us a chance to having stored object to delete + $clock = new MockClock(new \DateTimeImmutable('2024-01-01 00:00:00', new \DateTimeZone('+00:00'))); + $repository = $this->createMock(StoredObjectVersionRepository::class); + $repository->expects($this->once()) + ->method('findIdsByVersionsOlderThanDateAndNotLastVersion') + ->with(new \DateTime('2023-10-03 00:00:00', new \DateTimeZone('+00:00'))) + ->willReturnCallback(function ($arg) { + yield 1; + yield 3; + yield 2; + }) + ; + + $cronJob = new RemoveOldVersionCronJob($clock, $this->buildMessageBus(true), $repository); + + $results = $cronJob->run([]); + + self::assertArrayHasKey('last-deleted-stored-object-version-id', $results); + self::assertIsInt($results['last-deleted-stored-object-version-id']); + } + + public static function buildTestCanRunData(): iterable + { + yield [ + (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-31 00:00:00', new \DateTimeZone('+00:00'))), + true, + ]; + + yield [ + (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-30 23:59:59', new \DateTimeZone('+00:00'))), + true, + ]; + + yield [ + (new CronJobExecution('last-deleted-stored-object-version-id'))->setLastEnd(new \DateTimeImmutable('2023-12-31 00:00:01', new \DateTimeZone('+00:00'))), + false, + ]; + + yield [ + null, + true, + ]; + } + + private function buildMessageBus(bool $expectDistpatchAtLeastOnce = false): MessageBusInterface + { + $messageBus = $this->createMock(MessageBusInterface::class); + + $methodDispatch = match ($expectDistpatchAtLeastOnce) { + true => $messageBus->expects($this->atLeastOnce())->method('dispatch')->with($this->isInstanceOf(RemoveOldVersionMessage::class)), + false => $messageBus->method('dispatch'), + }; + + $methodDispatch->willReturnCallback(function (RemoveOldVersionMessage $message) { + return new Envelope($message); + }); + + return $messageBus; + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php new file mode 100644 index 000000000..109453bb7 --- /dev/null +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectCleaner/RemoveOldVersionMessageHandlerTest.php @@ -0,0 +1,51 @@ +registerVersion(); + $storedObjectVersionRepository = $this->createMock(StoredObjectVersionRepository::class); + $storedObjectVersionRepository->expects($this->once())->method('find') + ->with($this->identicalTo(1)) + ->willReturn($version); + + $entityManager = $this->createMock(EntityManagerInterface::class); + $entityManager->expects($this->once())->method('remove')->with($this->identicalTo($version)); + $entityManager->expects($this->once())->method('flush'); + $entityManager->expects($this->once())->method('clear'); + + $storedObjectManager = $this->createMock(StoredObjectManagerInterface::class); + $storedObjectManager->expects($this->once())->method('delete')->with($this->identicalTo($version)); + + $handler = new RemoveOldVersionMessageHandler($storedObjectVersionRepository, new NullLogger(), $entityManager, $storedObjectManager); + + $handler(new RemoveOldVersionMessage(1)); + } +} diff --git a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php index 3725f910f..4bb370ba3 100644 --- a/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php +++ b/src/Bundle/ChillDocStoreBundle/Tests/Service/StoredObjectManagerTest.php @@ -198,6 +198,37 @@ final class StoredObjectManagerTest extends TestCase self::assertSame($storedObject->getCurrentVersion(), $newVersion); } + public function testDelete(): void + { + $storedObject = new StoredObject(); + $version = $storedObject->registerVersion(filename: 'object_name'); + + $httpClient = new MockHttpClient(function ($method, $url, $options) { + self::assertEquals('DELETE', $method); + self::assertEquals('https://example.com/object_name', $url); + + return new MockResponse('', [ + 'http_code' => 204, + ]); + }); + + $tempUrlGenerator = $this->createMock(TempUrlGeneratorInterface::class); + $tempUrlGenerator + ->expects($this->once()) + ->method('generate') + ->with($this->identicalTo('DELETE'), $this->identicalTo('object_name')) + ->willReturnCallback(function (string $method, string $objectName) { + return new SignedUrl( + $method, + 'https://example.com/'.$objectName, + new \DateTimeImmutable('1 hours') + ); + }); + + $storedObjectManager = new StoredObjectManager($httpClient, $tempUrlGenerator); + $storedObjectManager->delete($version); + } + public function testWriteWithDeleteAt() { $storedObject = new StoredObject();