Introduce isort and a Makefile for simpler formatting, etc (#286)

* isort and makefile with useful targets

* update readme

* remove erroneously-committed (safe) files

* add isort to workflow yaml

* isort profile

.github/workflows/lint.yml

@@ -1,4 +1,4 @@
-name: Black, Ruff and MyPy Checks
+name: ruff, isort, black and mypy checks
 
 on:
   push:
@@ -23,11 +23,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install poetry
-        poetry install
-  
-    - name: Black linting
-      run: |
-        poetry run black --check .
+        poetry install --with dev
 
     - name: Ruff Linting AstraPy
       run: |
@@ -37,6 +33,22 @@ jobs:
       run: |
         poetry run ruff tests
 
+    - name: Isort Linting AstraPy
+      run: |
+        poetry run isort --check astrapy --profile black
+
+    - name: Isort Linting Tests
+      run: |
+        poetry run isort --check tests --profile black
+  
+    - name: Black linting astrapy
+      run: |
+        poetry run black --check astrapy
+  
+    - name: Black linting tests
+      run: |
+        poetry run black --check tests
+
     - name: Run MyPy AstraPy
       run: |
         poetry run mypy astrapy

Makefile

@@ -0,0 +1,57 @@
+SHELL := /bin/bash
+
+.PHONY: all format format-fix format-tests format-src test-idiomatic test-idiomatic-unit test-idiomatic-integration build help
+
+all: help
+
+FMT_FLAGS ?= --check
+
+format: format-src format-tests
+
+format-tests:
+	poetry run ruff tests
+	poetry run isort tests $(FMT_FLAGS) --profile black
+	poetry run black tests $(FMT_FLAGS)
+	poetry run mypy tests
+
+format-src:
+	poetry run ruff astrapy
+	poetry run isort astrapy $(FMT_FLAGS) --profile black
+	poetry run black astrapy $(FMT_FLAGS)
+	poetry run mypy astrapy
+
+format-fix: format-fix-src format-fix-tests
+
+format-fix-src: FMT_FLAGS=
+format-fix-src: format-src
+
+format-fix-tests: FMT_FLAGS=
+format-fix-tests: format-tests
+
+test-idiomatic: test-idiomatic-unit test-idiomatic-integration
+
+test-idiomatic-unit:
+	poetry run pytest tests/idiomatic/unit -vv
+
+test-idiomatic-integration:
+	poetry run pytest tests/idiomatic/integration -vv
+
+build:
+	rm -f dist/astrapy*
+	poetry build
+
+help:
+	@echo "======================================================================"
+	@echo "AstraPy make command             purpose"
+	@echo "----------------------------------------------------------------------"
+	@echo "format                           full lint and format checks"
+	@echo "  format-src                       limited to source"
+	@echo "  format-tests                     limited to tests"
+	@echo "  format-fix                       fixing imports and style"
+	@echo "    format-fix-src                   limited to source"
+	@echo "    format-fix-tests                 limited to tests"
+	@echo "test-idiomatic                   run idiomatic tests"
+	@echo "  test-idiomatic-unit              unit only"
+	@echo "  test-idiomatic-integration       integration only"
+	@echo "build                            build package ready for PyPI"
+	@echo "======================================================================"

README.md

@@ -206,11 +206,11 @@ First install poetry with `pip install poetry` and then the project dependencies
 Linter, style and typecheck should all pass for a PR:
 
 ```bash
-poetry run black --check astrapy && poetry run ruff astrapy && poetry run mypy astrapy
-
-poetry run black --check tests && poetry run ruff tests && poetry run mypy tests
+make format
 ```
 
+With `make format-fix` the style and imports are autofixed (by `black` and `isort` resp.)
+
 Features must be thoroughly covered in tests (see `tests/idiomatic/*` for
 naming convention and module structure).
 
@@ -227,7 +227,9 @@ export ASTRA_DB_KEYSPACE="default_keyspace"
 export ASTRA_DB_SECONDARY_KEYSPACE="..."
 ```
 
-Tests can be started in various ways:
+#### "Idiomatic" testing
+
+Tests can be started in various ways: mostly `make tests-idiomatic`, but also:
 
 ```bash
 # test the "idiomatic" layer
@@ -244,6 +246,8 @@ The (idiomatic) Admin part is tested manually by you, on Astra accounts with roo
 for up to 3 new databases, possibly both on prod and dev, and uses specific env vars,
 as can be seen on `tests/idiomatic/integration/test_admin.py`.
 
+#### Other tests
+
 Vectorize tests are confined in `tests/vectorize_idiomatic` and are run
 separately. A separate set of credentials is required to do the full testing:
 refer to `tests/.vectorize.env.template` for the complete listing, including

astrapy/__init__.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import toml
-import os
 import importlib.metadata
+import os
+
+import toml
 
 
 def get_version() -> str:
@@ -46,29 +47,20 @@ def get_version() -> str:
 __version__: str = get_version()
 
 
-# A circular-import issue requires this to happen at the end of this module:
-from astrapy.database import (  # noqa: E402
-    AsyncDatabase,
-    Database,
-)
-from astrapy.collection import (  # noqa: E402
-    AsyncCollection,
-    Collection,
-)
+import astrapy.constants  # noqa: E402
+import astrapy.cursors  # noqa: E402
+import astrapy.ids  # noqa: E402
+import astrapy.operations  # noqa: F401, E402
 from astrapy.admin import (  # noqa: E402
     AstraDBAdmin,
     AstraDBDatabaseAdmin,
     DataAPIDatabaseAdmin,
 )
-from astrapy.client import (  # noqa: E402
-    DataAPIClient,
-)
-
-import astrapy.ids  # noqa: E402
-import astrapy.constants  # noqa: E402
-import astrapy.cursors  # noqa: E402
-import astrapy.operations  # noqa: F401, E402
+from astrapy.client import DataAPIClient  # noqa: E402
+from astrapy.collection import AsyncCollection, Collection  # noqa: E402
 
+# A circular-import issue requires this to happen at the end of this module:
+from astrapy.database import AsyncDatabase, Database  # noqa: E402
 
 __all__ = [
     "AstraDBAdmin",

astrapy/admin.py

@@ -19,28 +19,27 @@
 import re
 import time
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
 from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 import httpx
 
-from astrapy.core.ops import AstraDBOps
-from astrapy.core.defaults import DEFAULT_AUTH_HEADER
 from astrapy.api_commander import APICommander
 from astrapy.authentication import coerce_token_provider
-from astrapy.cursors import CommandCursor
 from astrapy.constants import Environment
-from astrapy.info import AdminDatabaseInfo, DatabaseInfo
+from astrapy.core.defaults import DEFAULT_AUTH_HEADER
+from astrapy.core.ops import AstraDBOps
+from astrapy.cursors import CommandCursor
 from astrapy.exceptions import (
+    DataAPIFaultyResponseException,
     DevOpsAPIException,
     MultiCallTimeoutManager,
-    DataAPIFaultyResponseException,
     base_timeout_info,
-    to_dataapi_timeout_exception,
-    ops_recast_method_sync,
     ops_recast_method_async,
+    ops_recast_method_sync,
+    to_dataapi_timeout_exception,
 )
-
+from astrapy.info import AdminDatabaseInfo, DatabaseInfo
 
 if TYPE_CHECKING:
     from astrapy import AsyncDatabase, Database

astrapy/api/__init__.py

@@ -16,7 +16,6 @@
 
 from astrapy.core.api import APIRequestError
 
-
 __all__ = [
     "APIRequestError",
 ]

astrapy/api_commander.py

@@ -14,9 +14,9 @@
 
 from __future__ import annotations
 
+import json
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, cast
 
-import json
 import httpx
 
 from astrapy.core.defaults import (
@@ -26,25 +26,24 @@
     DEFAULT_VECTORIZE_SECRET_HEADER,
 )
 from astrapy.core.utils import (
+    TimeoutInfoWideType,
     http_methods,
-    logger,
     log_request,
     log_response,
+    logger,
     normalize_for_api,
     restore_from_api,
-    TimeoutInfoWideType,
     to_httpx_timeout,
     user_agent_astrapy,
     user_agent_rs,
     user_agent_string,
 )
 from astrapy.exceptions import (
-    DataAPIResponseException,
     DataAPIFaultyResponseException,
+    DataAPIResponseException,
     to_dataapi_timeout_exception,
 )
 
-
 DEFAULT_REDACTED_HEADER_NAMES = [
     DEFAULT_AUTH_HEADER,
     DEFAULT_DEV_OPS_AUTH_HEADER,

astrapy/api_options.py

@@ -17,7 +17,6 @@
 from dataclasses import dataclass
 from typing import Optional, TypeVar
 
-
 AO = TypeVar("AO", bound="BaseAPIOptions")
 
 

astrapy/client.py

@@ -16,7 +16,7 @@
 
 import logging
 import re
-from typing import Any, Dict, Optional, Union, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
 
 from astrapy.admin import (
     api_endpoint_parser,
@@ -29,7 +29,6 @@
 from astrapy.authentication import coerce_token_provider
 from astrapy.constants import Environment
 
-
 if TYPE_CHECKING:
     from astrapy import AsyncDatabase, Database
     from astrapy.admin import AstraDBAdmin

astrapy/collection.py

@@ -15,22 +15,31 @@
 from __future__ import annotations
 
 import asyncio
-import deprecation
 import json
 import logging
 from concurrent.futures import ThreadPoolExecutor
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union
+
+import deprecation
 
 from astrapy import __version__
-from astrapy.core.db import (
-    AstraDBCollection,
-    AsyncAstraDBCollection,
+from astrapy.api_options import CollectionAPIOptions
+from astrapy.constants import (
+    DocumentType,
+    FilterType,
+    ProjectionType,
+    ReturnDocument,
+    SortType,
+    VectorType,
+    normalize_optional_projection,
 )
+from astrapy.core.db import AstraDBCollection, AsyncAstraDBCollection
 from astrapy.core.defaults import (
     DEFAULT_INSERT_NUM_DOCUMENTS,
     DEFAULT_VECTORIZE_SECRET_HEADER,
 )
-from astrapy.api_options import CollectionAPIOptions
+from astrapy.cursors import AsyncCursor, Cursor
+from astrapy.database import AsyncDatabase, Database
 from astrapy.exceptions import (
     BulkWriteException,
     CollectionNotFoundException,
@@ -42,31 +51,19 @@
     MultiCallTimeoutManager,
     TooManyDocumentsToCountException,
     UpdateManyException,
-    recast_method_sync,
-    recast_method_async,
     base_timeout_info,
+    recast_method_async,
+    recast_method_sync,
 )
-from astrapy.constants import (
-    DocumentType,
-    FilterType,
-    ProjectionType,
-    ReturnDocument,
-    SortType,
-    VectorType,
-    normalize_optional_projection,
-)
-from astrapy.database import AsyncDatabase, Database
+from astrapy.info import CollectionInfo, CollectionOptions
+from astrapy.meta import check_deprecated_vector_ize
 from astrapy.results import (
+    BulkWriteResult,
     DeleteResult,
     InsertManyResult,
     InsertOneResult,
     UpdateResult,
-    BulkWriteResult,
 )
-from astrapy.cursors import AsyncCursor, Cursor
-from astrapy.info import CollectionInfo, CollectionOptions
-from astrapy.meta import check_deprecated_vector_ize
-
 
 if TYPE_CHECKING:
     from astrapy.operations import AsyncBaseOperation, BaseOperation

astrapy/constants.py

@@ -16,7 +16,6 @@
 
 from typing import Any, Dict, Iterable, Optional, Union
 
-
 DocumentType = Dict[str, Any]
 # ["field1", "field2"] allowed, but also:
 # {"field": True/False}

astrapy/core/api.py

@@ -15,9 +15,10 @@
 from __future__ import annotations
 
 import logging
-import httpx
 from typing import Any, Dict, Optional, Union, cast
 
+import httpx
+
 from astrapy.core.core_types import API_RESPONSE
 from astrapy.core.utils import amake_request, make_request
 

astrapy/core/core_types.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from __future__ import annotations
+
 from typing import Any, Dict, List, Protocol, Union
 
 # A type for: