From 93e7d0a8a7af343f3090cfc33a899f5c11da267f Mon Sep 17 00:00:00 2001 From: Greg Pascale Date: Tue, 4 Jun 2024 09:14:27 -0700 Subject: [PATCH 1/2] add filesystem_storage, a filesystem-based backend for artifact storage --- aim/storage/artifacts/artifact_registry.py | 2 ++ aim/storage/artifacts/filesystem_storage.py | 39 +++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 aim/storage/artifacts/filesystem_storage.py diff --git a/aim/storage/artifacts/artifact_registry.py b/aim/storage/artifacts/artifact_registry.py index bcd1ebc23..91ea4e2b1 100644 --- a/aim/storage/artifacts/artifact_registry.py +++ b/aim/storage/artifacts/artifact_registry.py @@ -3,6 +3,7 @@ from typing import Type, Dict from .s3_storage import S3ArtifactStorage +from .filesystem_storage import FilesystemArtifactStorage from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -29,3 +30,4 @@ def get_storage(self, url: str) -> 'AbstractArtifactStorage': registry = ArtifactStorageRegistry() registry.register('s3', S3ArtifactStorage) +registry.register('file', FilesystemArtifactStorage) diff --git a/aim/storage/artifacts/filesystem_storage.py b/aim/storage/artifacts/filesystem_storage.py new file mode 100644 index 000000000..e292cfdba --- /dev/null +++ b/aim/storage/artifacts/filesystem_storage.py @@ -0,0 +1,39 @@ +import pathlib +import os +import shutil +import tempfile + +from urllib.parse import urlparse +from typing import Optional + +from .artifact_storage import AbstractArtifactStorage + + +class FilesystemArtifactStorage(AbstractArtifactStorage): + def __init__(self, url: str): + super().__init__(url) + res = urlparse(self.url) + path = res.path + self._prefix = path + + def upload_artifact(self, file_path: str, artifact_path: str, block: bool = False): + dest_path = pathlib.Path(self._prefix) / artifact_path + dest_dir = os.path.dirname(dest_path) + os.makedirs(dest_dir, exist_ok=True) + shutil.copy(file_path, dest_path) + + def download_artifact(self, artifact_path: str, dest_dir: Optional[str] = None) -> str: + if dest_dir is None: + dest_dir = pathlib.Path(tempfile.mkdtemp()) + else: + dest_dir = pathlib.Path(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + source_path = dest_path = pathlib.Path(self._prefix) / artifact_path + dest_path = dest_dir / source_path.name + shutil.copy(source_path, dest_path) + + return dest_path.as_posix() + + def delete_artifact(self, artifact_path: str): + path = pathlib.Path(self._prefix) / artifact_path + shutil.rmtree(path) From 54aea9a1c37e77c1e62cbea6dae0a1fef2557bd5 Mon Sep 17 00:00:00 2001 From: Albert Torosyan Date: Fri, 14 Jun 2024 11:06:05 +0400 Subject: [PATCH 2/2] [doc] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7aa6fede..2f925354f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Enhancements: - Add feature to delete full experiments (mauricekraus) - Add support for python 3.12 (mahnerak) +- Add filesystem-based backend for artifact storage (gpascale) ### Fixes: - Increase websockets max_size for large images sent to server (jasonmads)