Improve methods to cleanup collections and buckets (#119)

.pre-commit-config.yaml

@@ -48,7 +48,7 @@ repos:
       - id: no-commit-to-branch
         args: [--branch, dev, --branch, int, --branch, main]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.1
+    rev: v0.5.2
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --exclude, scripts]

.pyproject_generation/pyproject_custom.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hexkit"
-version = "3.3.0"
+version = "3.4.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 requires-python = ">=3.9"
 classifiers = [

pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "3.3.0"
+version = "3.4.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
     "pydantic >=2, <3",

src/hexkit/protocols/objstorage.py

@@ -457,7 +457,10 @@ async def _delete_object(self, *, bucket_id: str, object_id: str) -> None:
 
     # Validation logic for input parameter:
     # (is typically only used by the protocol but may also be used in
-    # provider-specific code)
+    # provider-specific code or overwritten by the provider)
+
+    _re_bucket_id = re.compile(r"^[a-z0-9\-]{3,63}$")
+    _re_bucket_id_msg = "must consist of 3-63 lowercase letters, digits or hyphens"
 
     @classmethod
     def _validate_bucket_id(cls, bucket_id: str):
@@ -466,43 +469,43 @@ def _validate_bucket_id(cls, bucket_id: str):
         https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
         Raises BucketIdValidationError if not valid.
         """
-        if not 3 <= len(bucket_id) < 64:
+        if not cls._re_bucket_id.match(bucket_id):
             raise cls.BucketIdValidationError(
                 bucket_id=bucket_id,
-                reason="must be between 3 and 63 characters long",
-            )
-        if not re.match(r"^[a-z0-9\-]*$", bucket_id):
-            raise cls.BucketIdValidationError(
-                bucket_id=bucket_id,
-                reason="only lowercase letters, digits and hyphens (-) are allowed",
+                reason=cls._re_bucket_id_msg,
             )
         if bucket_id.startswith("-") or bucket_id.endswith("-"):
             raise cls.BucketIdValidationError(
                 bucket_id=bucket_id,
-                reason="may not start or end with a hyphen (-).",
+                reason="may not start or end with a hyphen",
             )
 
+    _re_object_id = re.compile(r"^[a-zA-Z0-9\-_.]{3,255}$")
+    _re_object_id_msg = (
+        "must consist of 3-255 letters, digits, hyphens, underscores or dots"
+    )
+
     @classmethod
     def _validate_object_id(cls, object_id: str):
-        """Check whether a object id follows the recommended naming pattern.
+        """Check whether an object id follows the recommended naming pattern.
         This is roughly based on (plus some additional restrictions):
         https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
         Raises ObjectIdValidationError if not valid.
+
+        Note that the default implementation does not allow object IDs to contain
+        forward slashes (/) which are sometimes used to represent directories.
+        Providers can override this method or the above constants to allow slashes,
+        other characters or longer object names.
         """
-        if not 3 <= len(object_id) < 64:
-            raise cls.ObjectIdValidationError(
-                object_id=object_id,
-                reason="must be between 3 and 63 characters long",
-            )
-        if not re.match(r"^[a-zA-Z0-9\-\.]*$", object_id):
+        if not cls._re_object_id.match(object_id):
             raise cls.ObjectIdValidationError(
                 object_id=object_id,
-                reason="only letters, digits, hyphens (-) and dots (.) are allowed",
+                reason=cls._re_object_id_msg,
             )
         if object_id.startswith(("-", ".")) or object_id.endswith(("-", ".")):
             raise cls.ObjectIdValidationError(
                 object_id=object_id,
-                reason="may not start or end with a hyphen (-) or a dot (.).",
+                reason="object names may not start or end with a hyphen or a dot",
             )
 
     # Exceptions that may be used by implementation:

src/hexkit/providers/akafka/testutils.py

@@ -430,6 +430,7 @@ def record_events(
 
     async def clear_topics(
         self,
+        *,
         topics: Optional[Union[str, list[str]]] = None,
         exclude_internal: bool = True,
     ):

src/hexkit/providers/mongodb/testutils.py

@@ -61,24 +61,30 @@ class MongoDbFixture:
 
     def empty_collections(
         self,
+        *,
+        collections: Optional[Union[str, list[str]]] = None,
         exclude_collections: Optional[Union[str, list[str]]] = None,
     ):
-        """Drop all mongodb collections in the database.
+        """Drop the given mongodb collection(s) in the database.
 
+        If no collections are specified, all collections will be dropped.
         You can also specify collection(s) that should be excluded
         from the operation, i.e. collections that should be kept.
         """
         db_name = self.config.db_name
-        if exclude_collections is None:
-            exclude_collections = []
-        if isinstance(exclude_collections, str):
-            exclude_collections = [exclude_collections]
-        excluded_collections = set(exclude_collections)
         try:
-            collection_names = self.client[db_name].list_collection_names()
-            for collection_name in collection_names:
-                if collection_name not in excluded_collections:
-                    self.client[db_name].drop_collection(collection_name)
+            if collections is None:
+                collections = self.client[db_name].list_collection_names()
+            elif isinstance(collections, str):
+                collections = [collections]
+            if exclude_collections is None:
+                exclude_collections = []
+            elif isinstance(exclude_collections, str):
+                exclude_collections = [exclude_collections]
+            excluded_collections = set(exclude_collections)
+            for collection in collections:
+                if collection not in excluded_collections:
+                    self.client[db_name].drop_collection(collection)
         except (ExecutionTimeout, OperationFailure) as error:
             raise RuntimeError(
                 f"Could not drop collection(s) of Mongo database {db_name}"

src/hexkit/providers/s3/testutils/_fixtures.py

@@ -24,7 +24,7 @@
 from contextlib import contextmanager
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Any, NamedTuple, Optional
+from typing import Any, NamedTuple, Optional, Union
 
 try:
     from typing import Self
@@ -93,11 +93,10 @@ def __init__(self, config: S3Config, storage: S3ObjectStorage):
         self.config = config
         self.storage = storage
 
-    def get_buckets(self) -> set[str]:
+    def get_buckets(self) -> list[str]:
         """Return a list of the buckets currently existing in the S3 object storage."""
         response = self.storage._client.list_buckets()
-        buckets = {bucket["Name"] for bucket in response["Buckets"]}
-        return buckets
+        return [bucket["Name"] for bucket in response["Buckets"]]
 
     async def populate_buckets(self, buckets: list[str]):
         """Populate the storage with buckets."""
@@ -111,19 +110,66 @@ async def populate_file_objects(self, file_objects: list[FileObject]):
             self.storage, bucket_fixtures=[], object_fixtures=file_objects
         )
 
-    async def empty_buckets(self, buckets_to_exclude: Optional[list[str]] = None):
-        """Clean the test artifacts or files from the populated buckets."""
-        for bucket in self.get_buckets().difference(buckets_to_exclude or []):
+    async def empty_buckets(
+        self,
+        *,
+        buckets: Optional[Union[str, list[str]]] = None,
+        exclude_buckets: Optional[Union[str, list[str]]] = None,
+    ):
+        """Remove all test objects from the given bucket(s).
+
+        If no buckets are specified, all existing buckets will be emptied,
+        therefore we recommend to always specify a bucket list.
+        You can also specify bucket(s) that should be excluded
+        from the operation, i.e. buckets that should not be emptied.
+        """
+        if buckets is None:
+            buckets = self.get_buckets()
+        elif isinstance(buckets, str):
+            buckets = [buckets]
+        if exclude_buckets is None:
+            exclude_buckets = []
+        elif isinstance(exclude_buckets, str):
+            exclude_buckets = [exclude_buckets]
+        excluded_buckets = set(exclude_buckets)
+        for bucket in buckets:
+            if bucket in excluded_buckets:
+                continue
             # Get list of all objects in the bucket
             object_ids = await self.storage.list_all_object_ids(bucket_id=bucket)
             # Delete all of these objects
             for object_id in object_ids:
+                # Note that id validation errors can be raised here if the bucket
+                # was populated by other means than the S3 storage fixture.
+                # We intentionally do not catch these errors because they
+                # usually mean that you're operating on the wrong buckets.
                 await self.storage.delete_object(bucket_id=bucket, object_id=object_id)
 
-    async def delete_buckets(self, buckets_to_exclude: Optional[list[str]] = None):
-        """Delete the populated buckets."""
-        for bucket in self.get_buckets().difference(buckets_to_exclude or []):
-            await self.storage.delete_bucket(bucket, delete_content=True)
+    async def delete_buckets(
+        self,
+        *,
+        buckets: Optional[Union[str, list[str]]] = None,
+        exclude_buckets: Optional[Union[str, list[str]]] = None,
+    ):
+        """Delete the given bucket(s).
+
+        If no buckets are specified, all existing buckets will be deleted,
+        therefore we recommend to always specify a bucket list.
+        You can also specify bucket(s) that should be excluded
+        from the operation, i.e. buckets that should be kept.
+        """
+        if buckets is None:
+            buckets = self.get_buckets()
+        elif isinstance(buckets, str):
+            buckets = [buckets]
+        if exclude_buckets is None:
+            exclude_buckets = []
+        elif isinstance(exclude_buckets, str):
+            exclude_buckets = [exclude_buckets]
+        excluded_buckets = set(exclude_buckets)
+        for bucket in buckets:
+            if bucket not in excluded_buckets:
+                await self.storage.delete_bucket(bucket, delete_content=True)
 
     async def get_initialized_upload(self) -> UploadDetails:
         """Initialize a new empty multipart upload process.

tests/integration/test_mongodb.py

@@ -126,17 +126,22 @@ async def test_dao_find_all_without_collection(mongodb: MongoDbFixture):
 async def test_empty_collections(mongodb: MongoDbFixture):
     """Make sure mongo reset function works"""
     db = mongodb.client[mongodb.config.db_name]
-    db.create_collection("test1")
-    db.create_collection("test2")
-    assert len(db.list_collection_names()) == 2
+    for i in range(1, 4):
+        db.create_collection(f"test{i}")
 
-    mongodb.empty_collections(exclude_collections=["test1"])
-    assert db.list_collection_names() == ["test1"]
+    assert len(db.list_collection_names()) == 3
+
+    mongodb.empty_collections(collections=["test3"])
+    assert set(db.list_collection_names()) == {"test1", "test2"}
+    mongodb.empty_collections(exclude_collections=["test2"])
+    assert db.list_collection_names() == ["test2"]
+    mongodb.empty_collections()
+    assert db.list_collection_names() == []
 
 
 async def test_dao_happy(mongodb: MongoDbFixture):
     """Test the happy path of performing basic CRUD database interactions using
-    the MongoDbDaoFactory in a surrograte ID setting.
+    the MongoDbDaoFactory in a surrogate ID setting.
     """
     dao = await mongodb.dao_factory.get_dao(
         name="example",

tests/integration/test_s3.py

@@ -49,7 +49,9 @@ async def test_empty_buckets(s3: S3Fixture, tmp_file: FileObject):  # noqa: F811
     await s3.populate_file_objects(file_objects=[tmp_file])
 
     # test empty_buckets() with and without parameters
-    await s3.empty_buckets(buckets_to_exclude=[bucket_id])
+    await s3.empty_buckets(buckets=[])
+    assert await s3.storage.does_object_exist(bucket_id=bucket_id, object_id=object_id)
+    await s3.empty_buckets(exclude_buckets=[bucket_id])
     assert await s3.storage.does_object_exist(bucket_id=bucket_id, object_id=object_id)
     await s3.empty_buckets()
     assert not await s3.storage.does_object_exist(
@@ -64,7 +66,9 @@ async def test_delete_buckets(s3: S3Fixture, tmp_file: FileObject):  # noqa: F81
     await s3.populate_file_objects(file_objects=[tmp_file])
 
     # test delete_buckets() with and without parameters
-    await s3.delete_buckets(buckets_to_exclude=[bucket_id])
+    await s3.delete_buckets(buckets=[])
+    assert await s3.storage.does_bucket_exist(bucket_id=bucket_id)
+    await s3.delete_buckets(exclude_buckets=[bucket_id])
     assert await s3.storage.does_bucket_exist(bucket_id=bucket_id)
     await s3.delete_buckets()
     assert not await s3.storage.does_bucket_exist(bucket_id=bucket_id)