Skip to content

Commit

Permalink
🏗️(project) migrate to pydantic v2 and switch tests to polyfactory
Browse files Browse the repository at this point in the history
Migrating to `pydantic` v2 should speed up processing and allow
interoperability with projects such as `warren`. This migration makes the
hypothesis package used in tests obsolete, which is why we introduce
`polyfactory`.
  • Loading branch information
Leobouloc committed Jan 24, 2024
1 parent e55f98f commit 548bcc5
Show file tree
Hide file tree
Showing 138 changed files with 2,316 additions and 2,001 deletions.
1 change: 0 additions & 1 deletion docs/CHANGELOG.md

This file was deleted.

1 change: 1 addition & 0 deletions docs/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../CHANGELOG.md
1 change: 0 additions & 1 deletion docs/LICENSE.md

This file was deleted.

1 change: 1 addition & 0 deletions docs/LICENSE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../LICENSE.md
2 changes: 1 addition & 1 deletion docs/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
::: mkdocs-click
:module: ralph.cli
:command: cli
:depth: 1
:depth: 1
5 changes: 3 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ dependencies = [
# By default, we only consider core dependencies required to use Ralph as a
# library (mostly models).
"langcodes>=3.2.0",
"pydantic[dotenv,email]>=1.10.0, <2.0",
"pydantic[email]>=2.5.3,<3.0",
"pydantic_settings>=2.1.0,<3.0",
"rfc3987>=1.3.0",
]
dynamic = ["version"]
Expand Down Expand Up @@ -91,7 +92,6 @@ dev = [
"black==23.12.1",
"cryptography==41.0.7",
"factory-boy==3.3.0",
"hypothesis<6.92.0", # pin as hypothesis 6.92.0 observability feature seems broken
"logging-gelf==0.0.31",
"mike==2.0.0",
"mkdocs==1.5.3",
Expand All @@ -103,6 +103,7 @@ dev = [
"neoteroi-mkdocs==1.0.4",
"pyfakefs==5.3.2",
"pymdown-extensions==10.7",
"polyfactory==2.14.1",
"pytest==7.4.4",
"pytest-asyncio==0.23.3",
"pytest-cov==4.1.0",
Expand Down
5 changes: 4 additions & 1 deletion src/ralph/api/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,4 +50,7 @@ async def whoami(
user: AuthenticatedUser = Depends(get_authenticated_user),
) -> Dict[str, Any]:
"""Return the current user's username along with their scopes."""
return {"agent": user.agent, "scopes": user.scopes}
return {
"agent": user.agent.model_dump(mode="json", exclude_none=True),
"scopes": user.scopes,
}
32 changes: 16 additions & 16 deletions src/ralph/api/auth/basic.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Basic authentication & authorization related tools for the Ralph API."""

import logging
import os
from functools import lru_cache
from pathlib import Path
from threading import Lock
Expand All @@ -10,7 +11,7 @@
from cachetools import TTLCache, cached
from fastapi import Depends, HTTPException, status
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from pydantic import BaseModel, root_validator
from pydantic import RootModel, model_validator
from starlette.authentication import AuthenticationError

from ralph.api.auth.user import AuthenticatedUser
Expand Down Expand Up @@ -40,45 +41,42 @@ class UserCredentials(AuthenticatedUser):
username: str


class ServerUsersCredentials(BaseModel):
class ServerUsersCredentials(RootModel[List[UserCredentials]]):
"""Custom root pydantic model.
Describe expected list of all server users credentials as stored in
the credentials file.
Attributes:
__root__ (List): Custom root consisting of the
root (List): Custom root consisting of the
list of all server users credentials.
"""

__root__: List[UserCredentials]

def __add__(self, other) -> Any: # noqa: D105
return ServerUsersCredentials.parse_obj(self.__root__ + other.__root__)
return ServerUsersCredentials.model_validate(self.root + other.root)

def __getitem__(self, item: int) -> UserCredentials: # noqa: D105
return self.__root__[item]
return self.root[item]

def __len__(self) -> int: # noqa: D105
return len(self.__root__)
return len(self.root)

def __iter__(self) -> Iterator[UserCredentials]: # noqa: D105
return iter(self.__root__)
return iter(self.root)

@root_validator
@classmethod
def ensure_unique_username(cls, values: Any) -> Any:
@model_validator(mode="after")
def ensure_unique_username(self) -> Any:
"""Every username should be unique among registered users."""
usernames = [entry.username for entry in values.get("__root__")]
usernames = [entry.username for entry in self.root]
if len(usernames) != len(set(usernames)):
raise ValueError(
"You cannot create multiple credentials with the same username"
)
return values
return self


@lru_cache()
def get_stored_credentials(auth_file: Path) -> ServerUsersCredentials:
def get_stored_credentials(auth_file: os.PathLike) -> ServerUsersCredentials:
"""Helper to read the credentials/scopes file.
Read credentials from JSON file and stored them to avoid reloading them with every
Expand All @@ -96,7 +94,9 @@ def get_stored_credentials(auth_file: Path) -> ServerUsersCredentials:
msg = "Credentials file <%s> not found."
logger.warning(msg, auth_file)
raise AuthenticationError(msg.format(auth_file))
return ServerUsersCredentials.parse_file(auth_file)

with open(auth_file, encoding=settings.LOCALE_ENCODING) as f:
return ServerUsersCredentials.model_validate_json(f.read())


@cached(
Expand Down
12 changes: 5 additions & 7 deletions src/ralph/api/auth/oidc.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from fastapi.security import HTTPBearer, OpenIdConnect
from jose import ExpiredSignatureError, JWTError, jwt
from jose.exceptions import JWTClaimsError
from pydantic import AnyUrl, BaseModel, Extra
from pydantic import AnyUrl, BaseModel, ConfigDict
from typing_extensions import Annotated

from ralph.api.auth.user import AuthenticatedUser, UserScopes
Expand Down Expand Up @@ -44,13 +44,11 @@ class IDToken(BaseModel):

iss: str
sub: str
aud: Optional[str]
aud: Optional[str] = None
exp: int
iat: int
scope: Optional[str]

class Config: # noqa: D106
extra = Extra.ignore
scope: Optional[str] = None
model_config = ConfigDict(extra="ignore")


@lru_cache()
Expand Down Expand Up @@ -142,7 +140,7 @@ def get_oidc_user(
headers={"WWW-Authenticate": "Bearer"},
) from exc

id_token = IDToken.parse_obj(decoded_token)
id_token = IDToken.model_validate(decoded_token)

user = AuthenticatedUser(
agent={"openid": f"{id_token.iss}/{id_token.sub}"},
Expand Down
20 changes: 7 additions & 13 deletions src/ralph/api/auth/user.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
"""Authenticated user for the Ralph API."""

from typing import Dict, FrozenSet, Literal
from typing import FrozenSet, Literal

from pydantic import BaseModel
from pydantic import BaseModel, RootModel

from ralph.models.xapi.base.agents import BaseXapiAgent

Scope = Literal[
"statements/write",
Expand All @@ -18,7 +20,7 @@
]


class UserScopes(FrozenSet[Scope]):
class UserScopes(RootModel[FrozenSet[Scope]]):
"""Scopes available to users."""

def is_authorized(self, requested_scope: Scope):
Expand Down Expand Up @@ -47,19 +49,11 @@ def is_authorized(self, requested_scope: Scope):
}

expanded_user_scopes = set()
for scope in self:
for scope in self.root:
expanded_user_scopes.update(expanded_scopes.get(scope, {scope}))

return requested_scope in expanded_user_scopes

@classmethod
def __get_validators__(cls): # noqa: D105
def validate(value: FrozenSet[Scope]):
"""Transform value to an instance of UserScopes."""
return cls(value)

yield validate


class AuthenticatedUser(BaseModel):
"""Pydantic model for user authentication.
Expand All @@ -69,5 +63,5 @@ class AuthenticatedUser(BaseModel):
scopes (list): The scopes the user has access to.
"""

agent: Dict
agent: BaseXapiAgent
scopes: UserScopes
6 changes: 3 additions & 3 deletions src/ralph/api/forwarding.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,16 @@ async def forward_xapi_statements(
try:
# NB: post or put
req = await getattr(client, method)(
forwarding.url,
str(forwarding.url),
json=statements,
auth=(forwarding.basic_username, forwarding.basic_password),
timeout=forwarding.timeout,
)
req.raise_for_status()
msg = "Forwarded %s statements to %s with success."
if isinstance(statements, list):
logger.debug(msg, len(statements), forwarding.url)
logger.debug(msg, len(statements), str(forwarding.url))
else:
logger.debug(msg, 1, forwarding.url)
logger.debug(msg, 1, str(forwarding.url))
except (RequestError, HTTPStatusError) as error:
logger.error("Failed to forward xAPI statements. %s", error)
12 changes: 3 additions & 9 deletions src/ralph/api/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from typing import Optional, Union
from uuid import UUID

from pydantic import AnyUrl, BaseModel, Extra
from pydantic import AnyUrl, BaseModel, ConfigDict

from ..models.xapi.base.agents import BaseXapiAgent
from ..models.xapi.base.groups import BaseXapiGroup
Expand All @@ -29,13 +29,7 @@ class BaseModelWithLaxConfig(BaseModel):
we receive statements through the API.
"""

class Config:
"""Enable extra properties.
Useful for not having to perform comprehensive validation.
"""

extra = Extra.allow
model_config = ConfigDict(extra="allow", coerce_numbers_to_str=True)


class LaxObjectField(BaseModelWithLaxConfig):
Expand Down Expand Up @@ -64,6 +58,6 @@ class LaxStatement(BaseModelWithLaxConfig):
"""

actor: Union[BaseXapiAgent, BaseXapiGroup]
id: Optional[UUID]
id: Optional[UUID] = None
object: LaxObjectField
verb: LaxVerbField
2 changes: 1 addition & 1 deletion src/ralph/api/routers/health.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ async def heartbeat(response: Response) -> Heartbeat:
Return a 200 if all checks are successful.
"""
statuses = Heartbeat.construct(
statuses = Heartbeat.model_construct(
database=await await_if_coroutine(BACKEND_CLIENT.status())
)
if not statuses.is_alive:
Expand Down
Loading

0 comments on commit 548bcc5

Please sign in to comment.