diff --git a/Dockerfile.docs b/Dockerfile.docs
index 39a87bf..ea52b86 100644
--- a/Dockerfile.docs
+++ b/Dockerfile.docs
@@ -1,4 +1,4 @@
-FROM python:3.10-buster
+FROM python:3.10-slim-bullseye
RUN apt-get update && apt-get install -y \
pandoc default-jre graphviz \
diff --git a/docs/dsms_sdk/dsms_config_schema.md b/docs/dsms_sdk/dsms_config_schema.md
index 7104c4f..6fc43d8 100644
--- a/docs/dsms_sdk/dsms_config_schema.md
+++ b/docs/dsms_sdk/dsms_config_schema.md
@@ -9,13 +9,15 @@ This section describes the configuration properties for the DSMS Python SDK.
| Field Name | Description | Type | Default | Property Namespace | Required/Optional | Environment Variable |
|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------|------------------------------|-------------------------|-------------------|-------------------------------|
| Host URL | URL of the DSMS instance to connect. | `AnyUrl` | Not Applicable | `host_url` | Required | `DSMS_HOST_URL` |
-| Request timeout | Timeout in seconds until the request to the DSMS is timed out. | `int` | `120` | `request_timeout` | Optional | `DSMS_REQUEST_TIMEOUT` |
-| SSL verify | Whether the SSL of the DSMS shall be verified during connection. | `bool` | `True` | `ssl_verify` | Optional | `DSMS_SSL_VERIFY` |
| Username | User name for connecting to the DSMS instance | `Optional[SecretStr]`| `None` | `username` | Optional | `DSMS_USERNAME` |
+| Client ID | If a service account is used to authenticate, this will proviode the Client ID in Keycloak | `Optional[SecretStr]`| `None` | `client_id` | Optional | `DSMS_CLIENT_ID`, `KEYCLOAK_DSMS_CLIENT_ID` or `KEYCLOAK_CLIENT_ID` |
+| Client Secret | If a service account is used to authenticate, this will proviode the Client Secret in Keycloak | `Optional[SecretStr]`| `None` | `client_secret` | Optional | `DSMS_CLIENT_SECRET`, `KEYCLOAK_DSMS_CLIENT_SECRET` or `KEYCLOAK_CLIENT_SECRET` |
| Password | Password for connecting to the DSMS instance | `Optional[SecretStr]`| `None` | `password` | Optional | `DSMS_PASSWORD` |
| Token | JWT bearer token for connecting to the DSMS instance | `Optional[SecretStr]`| `None` | `token` | Optional | `DSMS_TOKEN` |
| Ping Backend | Check whether the host is a DSMS instance or not. | `bool` | `True` | `ping_backend` | Optional | `DSMS_PING_BACKEND` |
| Auto fetch KTypes | Whether the KTypes of the DSMS should be fetched automatically when the session is started. They will be fetched if requested and cached in memory. | `bool` | `True` | `auto_fetch_ktypes` | Optional | `DSMS_AUTO_FETCH_KTYPES` |
+| Request timeout | Timeout in seconds until the request to the DSMS is timed out. | `int` | `120` | `request_timeout` | Optional | `DSMS_REQUEST_TIMEOUT` |
+| SSL verify | Whether the SSL of the DSMS shall be verified during connection. | `bool` | `True` | `ssl_verify` | Optional | `DSMS_SSL_VERIFY` |
| Auto refresh | Determines whether local objects like KItem, KType, and AppConfig should automatically update with the latest backend data after a successful commit. | `bool` | `True` | `auto_refresh` | Optional | `DSMS_AUTO_REFRESH` |
| Always refetch KTypes | Whether the KTypes of the DSMS should be refetched every time used in the SDK. This can be helpful if the SDK is integrated in a service and the KTypes are updated.
**WARNING**: This might lead to performance issues. | `bool` | `False` | `always_refetch_ktypes` | Optional | `DSMS_ALWAYS_REFETCH_KTYPES` |
| Strict validation | Whether the validation of custom properties shall be strict. Disabling this might be helpful when, for example, the schema of a KType has been changed and the custom properties are not compatible anymore and should be updated accordingly. | `bool` | `True` | `strict_validation` | Optional | `DSMS_STRICT_VALIDATION` |
@@ -32,6 +34,19 @@ This section describes the configuration properties for the DSMS Python SDK.
| Hide properties | Properties to hide while printing, e.g `{'external_links'}` | `Set[str]` | `{}` | `hide_properties` | Optional | `DSMS_HIDE_PROPERTIES` |
| Log level | Logging level | `str` | `None` | `log_level` | Optional | `DSMS_LOG_LEVEL` |
+## Authentication Priority Rules
+
+```{important}
+
+When multiple authentication methods are provided:
+
+* Service account credentials (`client_id` + `client_secret`) always take precedence over user credentials (`username` + `password`)
+* If any authentication credentials are provided alongside a token, the original `token` value will be overwritten with a new token obtained during the authentication process
+
+Summary: Service account → User credentials → Direct token (in order of priority)
+
+```
+
## Example Usage
```python
from dsms import DSMS
diff --git a/docs/dsms_sdk/dsms_kitem_schema.md b/docs/dsms_sdk/dsms_kitem_schema.md
index e3e5abe..4ad3854 100644
--- a/docs/dsms_sdk/dsms_kitem_schema.md
+++ b/docs/dsms_sdk/dsms_kitem_schema.md
@@ -19,7 +19,7 @@ The schema contains complex types and references, indicating an advanced usage s
| Updated At | Timestamp of when the KItem was updated. | Union[string, datetime] | `None` | `updated_at` | Automatically generated |
| Avatar | The avatar of the KItem. | Union[[Avatar](#avatar-fields), Dict[str, Any]] | `None` | `avatar` | Optional |
| Avatar Exists | Whether the KItem holds an avatar or not. | boolean | `False` | `avatar_exists` | Automatically generated |
-| KItemCustomPropertiesModel(#kitemcustompropertiesmodel) | A set of custom properties related to the KItem. | Any | `None` | `custom_properties`| Optional |
+| [KItemCustomPropertiesModel](#kitemcustompropertiesmodel) | A set of custom properties related to the KItem. | Any | `None` | `custom_properties`| Optional |
| Summary | A brief human-readable summary of the KItem | string | `None` | `summary` | Optional |
| Apps | A list of applications associated with the KItem | List[[App](#app-fields)] | `[ ]` | `apps` | Optional |
| Annotations | A list of annotations related to the KItem | List[[Annotation](#annotation-fields)] | `[ ]` | `annotations` | Optional |
diff --git a/docs/dsms_sdk/dsms_sdk.md b/docs/dsms_sdk/dsms_sdk.md
index 5df1a37..2bb1425 100644
--- a/docs/dsms_sdk/dsms_sdk.md
+++ b/docs/dsms_sdk/dsms_sdk.md
@@ -89,9 +89,10 @@ You need to authenticate yourself to connect with dsms using the `dsms-sdk` Pyth
The following are the instances of the DSMS you could choose from:
- - [StahlDigital](https://lnkd.in/gfwe9a36)
- - [KupferDigital](https://lnkd.in/g8mvnM3K)
- - [DiMAT](https://lnkd.in/g46baB6J)
+ - [StahlDigital](https://stahldigital.materials-data.space)
+ - [KupferDigital](https://kupferdigital.materials-data.space)
+ - [DiMAT](https://cmdb.materials-data.space)
+ - [ORCHESTER](https://cmdb.materials-data.space)
2. **Authentication Options**
@@ -160,6 +161,10 @@ You need to authenticate yourself to connect with dsms using the `dsms-sdk` Pyth
DSMS_TOKEN = {YOUR_COPIED_TOKEN}
```
+```{important}
+Please also note the [priority rules for authentication](dsms_config_schema.md#authentication-priority-rules).
+```
+
Now you are ready to use dsms-sdk. Do check out the tutorials section to try out some basic examples on how to use dsms-sdk.
The next sections covers about the schema of fundamental classes crucial for users to know about DSMS when using the platform. Below given explains about the Schema of `KItem` and its associated properties in DSMS.
diff --git a/dsms/core/configuration.py b/dsms/core/configuration.py
index 3e20ed0..3b842f6 100644
--- a/dsms/core/configuration.py
+++ b/dsms/core/configuration.py
@@ -8,7 +8,6 @@
import requests
-from pydantic_core.core_schema import ValidationInfo # isort: skip
from pydantic_settings import BaseSettings, SettingsConfigDict # isort: skip
@@ -18,24 +17,30 @@
ConfigDict,
Field,
SecretStr,
+ model_validator,
field_validator,
)
-from .utils import get_callable # isort: skip
+from dsms.core.utils import get_callable # isort: skip
+from dsms.core.logging import handler # isort: skip
MODULE_REGEX = r"^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*:[a-zA-Z_][a-zA-Z0-9_]*$"
DEFAULT_UNIT_SPARQL = "dsms.knowledge.semantics.units.sparql:UnitSparqlQuery"
DEFAULT_REPO = "knowledge-items"
+logger = logging.getLogger(__name__)
+logger.addHandler(handler)
+logger.propagate = False
+
class Loglevel(Enum):
"""Enum mapping for default log levels"""
- DEBUG: logging.DEBUG
- INFO: logging.INFO
- ERROR: logging.ERROR
- CRITICAL: logging.CRITICAL
- WARNING: logging.WARNING
+ DEBUG = logging.DEBUG
+ INFO = logging.INFO
+ ERROR = logging.ERROR
+ CRITICAL = logging.CRITICAL
+ WARNING = logging.WARNING
class Configuration(BaseSettings):
@@ -44,6 +49,48 @@ class Configuration(BaseSettings):
host_url: AnyUrl = Field(
..., description="Url of the DSMS instance to connect."
)
+
+ username: Optional[SecretStr] = Field(
+ None,
+ description="User name for connecting to the DSMS instance",
+ )
+ password: Optional[SecretStr] = Field(
+ None,
+ description="Password for connecting to the DSMS instance",
+ )
+
+ client_id: Optional[SecretStr] = Field(
+ None,
+ description="""If a service account is used to authenticate,
+ this will proviode the Client ID in Keycloak""",
+ validation_alias=AliasChoices(
+ "DSMS_CLIENT_ID", "KEYCLOAK_DSMS_CLIENT_ID", "KEYCLOAK_CLIENT_ID"
+ ),
+ )
+
+ client_secret: Optional[SecretStr] = Field(
+ None,
+ description="""If a service account is used to authenticate,
+ this will proviode the Client Secret in Keycloak""",
+ validation_alias=AliasChoices(
+ "DSMS_CLIENT_SECRET",
+ "KEYCLOAK_DSMS_CLIENT_SECRET",
+ "KEYCLOAK_CLIENT_SECRET",
+ ),
+ )
+
+ realm: Optional[SecretStr] = Field(
+ "dsms",
+ description="""When the Cliend ID and Client secret is used for authentication
+ with a service account, this is the realm name to be used""",
+ validation_alias=AliasChoices("DSMS_REALM", "KEYCLOAK_REALM_NAME"),
+ )
+
+ token: Optional[SecretStr] = Field(
+ None,
+ description="JWT bearer token for connecting to the DSMS instance",
+ )
+
request_timeout: int = Field(
120,
description="Timeout in seconds until the request to the DSMS is timed out.",
@@ -61,19 +108,6 @@ class Configuration(BaseSettings):
and the custom properties are not compatible anymore and should be updated accordingly.""",
)
- username: Optional[SecretStr] = Field(
- None,
- description="User name for connecting to the DSMS instance",
- )
- password: Optional[SecretStr] = Field(
- None,
- description="Password for connecting to the DSMS instance",
- )
- token: Optional[SecretStr] = Field(
- None,
- description="JWT bearer token for connecting to the DSMS instance",
- )
-
enable_auto_reauth: bool = Field(
True,
description="""Whether to automatically reauthenticate with username and password
@@ -139,6 +173,12 @@ class Configuration(BaseSettings):
description="Repository of the triplestore for KItems in the DSMS",
)
+ loglevel: Optional[Union[Loglevel, str]] = Field(
+ None,
+ description="Set level of logging messages",
+ alias=AliasChoices("loglevel", "log_level"),
+ )
+
qudt_units: AnyUrl = Field(
"http://qudt.org/2.1/vocab/unit",
description="URI to QUDT Unit ontology for unit conversion",
@@ -162,12 +202,6 @@ class Configuration(BaseSettings):
description="Properties to hide while printing, e.g {'external_links'}",
)
- loglevel: Optional[Union[Loglevel, str]] = Field(
- None,
- description="Set level of logging messages",
- alias=AliasChoices("loglevel", "log_level"),
- )
-
model_config = ConfigDict(use_enum_values=True)
@field_validator("loglevel")
@@ -177,6 +211,7 @@ def get_loglevel(
"""Set log level for package"""
if val:
logging.getLogger().setLevel(val)
+ logger.setLevel(val)
return val
@field_validator("units_sparql_object")
@@ -212,30 +247,51 @@ def validate_strictness(cls, val: bool) -> bool:
)
return val
- @field_validator("token")
- def validate_auth(cls, val, info: ValidationInfo):
+ @model_validator(mode="after")
+ def validate_auth(self):
"""Validate the provided authentication/authorization secrets."""
- username = info.data.get("username")
- passwd = info.data.get("password")
- host_url = info.data.get("host_url")
- timeout = info.data.get("request_timeout")
- verify = info.data.get("ssl_verify")
- if username and passwd and val:
- raise ValueError(
- "Either `username` and `password` or `token` must be provided. Not both."
+ username = self.username
+ passwd = self.password
+ host_url = self.host_url
+ client_id = self.client_id
+ client_secret = self.client_secret
+ realm = self.realm
+ timeout = self.request_timeout
+ verify = self.ssl_verify
+ val = self.token
+
+ if client_id and client_secret:
+ token_url = urllib.parse.urljoin(
+ str(host_url),
+ f"/auth/realms/{realm.get_secret_value()}/protocol/openid-connect/token", # pylint: disable=no-member
)
- if username and not passwd:
- raise ValueError("`username` provided, but `password` not.")
- if not username and passwd:
- raise ValueError("`password` but not the `username` is defined.")
- if not username and not passwd and not val:
- warnings.warn(
- """No authentication details provided. Either `username` and `password`
- or `token` must be provided."""
+ headers = {"Content-Type": "application/x-www-form-urlencoded"}
+ data = {
+ "grant_type": "client_credentials",
+ "client_id": client_id.get_secret_value(), # pylint: disable=no-member
+ "client_secret": client_secret.get_secret_value(), # pylint: disable=no-member
+ }
+ logger.debug("Sending post request to %s", token_url)
+ response = requests.post(
+ token_url,
+ headers=headers,
+ data=data,
+ timeout=timeout,
+ verify=verify,
+ )
+ if not response.ok:
+ raise RuntimeError(
+ f"Authentication with service account was not successful: {response.text}",
+ )
+ val = response.json().get("access_token")
+ logger.info(
+ "Authenticated with Client ID and Client Secret at %s",
+ host_url,
)
- if not val and username and passwd:
+ elif username and passwd:
url = urllib.parse.urljoin(str(host_url), "api/users/token")
- authorization = f"Basic {username.get_secret_value()}:{passwd.get_secret_value()}"
+ authorization = f"Basic {username.get_secret_value()}:{passwd.get_secret_value()}" # pylint: disable=no-member
+ logger.debug("Sending get request to %s", url)
response = requests.get(
url,
headers={"Authorization": authorization},
@@ -247,15 +303,46 @@ def validate_auth(cls, val, info: ValidationInfo):
f"Something went wrong fetching the access token: {response.text}"
)
val = response.json().get("token")
+ logger.info(
+ "Authenticated with User name and Password at %s", host_url
+ )
+ elif val:
+ logger.info(
+ "Authenticated using token copied from WebUI interface at %s",
+ host_url,
+ )
+ else:
+ provided = {
+ key: value is not None
+ for key, value in {
+ "client_id": client_id,
+ "client_secret": client_secret,
+ "username": username,
+ "password": passwd,
+ "token": val,
+ }.items()
+ }
+ warnings.warn(
+ f"""No authentication details provided - protected endpoints may be inaccessible.
+ The followings were provided: {provided}""",
+ )
+
if isinstance(val, str):
if "Bearer " not in val:
val = SecretStr(f"Bearer {val}")
else:
val = SecretStr(val)
elif isinstance(val, SecretStr):
- if "Bearer " not in val.get_secret_value():
- val = SecretStr(f"Bearer {val.get_secret_value()}")
+ if (
+ "Bearer "
+ not in val.get_secret_value() # pylint: disable=no-member
+ ):
+ val = SecretStr(
+ f"Bearer {val.get_secret_value()}" # pylint: disable=no-member
+ )
- return val
+ # Set the validated token value and return self
+ self.token = val
+ return self
model_config = SettingsConfigDict(env_prefix="DSMS_")
diff --git a/dsms/core/utils.py b/dsms/core/utils.py
index 9a9ca6d..38cd3e4 100644
--- a/dsms/core/utils.py
+++ b/dsms/core/utils.py
@@ -8,7 +8,6 @@
from uuid import UUID
import requests
-from pydantic import SecretStr
from requests import Response
from dsms.core.logging import handler # isort:skip
@@ -71,30 +70,11 @@ def _perform_request(
and dsms.config.enable_auto_reauth
and retry
):
- if dsms.config.username and dsms.config.password:
- username = dsms.config.username.get_secret_value()
- passwd = dsms.config.password.get_secret_value()
- authorization = f"Basic {username}:{passwd}"
- reauth = _perform_request(
- dsms,
- "api/users/token",
- "get",
- retry=False,
- headers={"Authorization": authorization},
- )
- if not reauth.ok:
- raise RuntimeError(f"Reauthentication failed: {reauth.text}")
- logger.debug("Reauthentication successful.")
- token = reauth.json().get("token")
- if "Bearer " not in token:
- dsms.config.token = SecretStr(f"Bearer {token}")
- else:
- dsms.config.token = SecretStr(token)
- response = _perform_request(
- dsms, route, method, retry=False, headers=None, **kwargs
- )
- else:
- logger.debug("No credentials found for reauthentication.")
+ logger.info("Token expired. Will re-authenticate.")
+ dsms.config = dsms.config.model_validate(dsms.config)
+ response = _perform_request(
+ dsms, route, method, retry=False, headers=None, **kwargs
+ )
else:
logger.debug(
"Reauthentication skipped. Either not needed or not enabled."
diff --git a/dsms/knowledge/properties/avatar.py b/dsms/knowledge/properties/avatar.py
index 0630af2..9b7bf6d 100644
--- a/dsms/knowledge/properties/avatar.py
+++ b/dsms/knowledge/properties/avatar.py
@@ -21,7 +21,7 @@ class Avatar(BaseModel):
description="The file path to the image when setting a new avatar is set",
)
encode_qr: Optional[str] = Field(
- False,
+ None,
description="""String for e.g. a link with should be encoded
into an QR code. This can be combined with an image file.""",
)
diff --git a/dsms/knowledge/properties/linked_kitems.py b/dsms/knowledge/properties/linked_kitems.py
index a654782..aec1078 100644
--- a/dsms/knowledge/properties/linked_kitems.py
+++ b/dsms/knowledge/properties/linked_kitems.py
@@ -33,7 +33,7 @@ class KItemRelationshipModel(BaseModel):
False,
description="""Whether the relation is incoming. This field is read-only.
Link with the field set to `true` are ignored during the commit""",
- allow_mutation=False,
+ frozen=True,
)
label: Optional[str] = Field(None, description="Label of the relation")
kitem: Union[KItemCompactedModel, KItemLinkedModel, Any] = Field(
diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py
index 2001625..40de85c 100644
--- a/dsms/knowledge/utils.py
+++ b/dsms/knowledge/utils.py
@@ -895,7 +895,7 @@ def _make_annotation_schema(iri: str) -> Dict[str, Any]:
def _search(
dsms: "DSMS",
query: Optional[str] = None,
- ktypes: "Optional[List[Union[Enum, KType]]]" = [],
+ ktypes: "Optional[Union[List[Union[Enum, KType]], Union[Enum,KType]]]" = [],
annotations: "Optional[List[str]]" = [],
limit: "Optional[int]" = 10,
offset: "Optional[int]" = 0,
@@ -905,6 +905,9 @@ def _search(
"""Search for KItems in the remote backend"""
from dsms import KItem, KItemCompactedModel
+ if not isinstance(ktypes, list):
+ ktypes = [ktypes]
+
payload = {
"search_term": query or "",
"ktypes": [ktype.value.id for ktype in ktypes],
diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py
index 84153d3..96986eb 100644
--- a/dsms/knowledge/webform.py
+++ b/dsms/knowledge/webform.py
@@ -192,6 +192,10 @@ class RelationMapping(BaseWebformModel):
None,
description="Target class IRI if the type of relation is an object property",
)
+ inverse: Optional[bool] = Field(
+ False,
+ description="If true, the relation is reversed",
+ )
def __str__(self) -> str:
"""Pretty print the model fields"""
@@ -283,6 +287,9 @@ class Webform(BaseWebformModel):
[], description="Class mapping"
)
sections: List[Section] = Field([], description="List of sections")
+ description: Optional[str] = Field(
+ None, description="Description of the webform"
+ )
def __str__(self) -> str:
"""Pretty print the model fields"""
diff --git a/setup.cfg b/setup.cfg
index 05eb9ea..ef86437 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -25,7 +25,7 @@ install_requires =
lru-cache<1
oyaml==1
pandas>=2,<3
- pydantic>=2,<3
+ pydantic>=2,<=2.11.7
pydantic-settings
python-dotenv
qrcode-artistic>=3,<4
@@ -59,7 +59,7 @@ pre_commit =
pre-commit==3.3.2
pylint==3.2.0
tests =
- pytest==6.2.5
+ pytest>=7.4.3
pytest-mock
responses
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 941915c..e988333 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -100,22 +100,24 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address):
}
],
}
-
- kitem_new = KItem(
- id=get_mock_kitem_ids[0],
- name="foo123",
- ktype_id=dsms.ktypes.Organization,
- annotations=[
- {
- "iri": "http://example.org/",
- "label": "foo",
- "namespace": "example",
- }
- ],
- linked_kitems=[linked_kitem3],
- user_groups=[user_group],
- apps=[app],
- )
+ with pytest.warns(
+ UserWarning, match="Found a "
+ ):
+ kitem_new = KItem(
+ id=get_mock_kitem_ids[0],
+ name="foo123",
+ ktype_id=dsms.ktypes.Organization,
+ annotations=[
+ {
+ "iri": "http://example.org/",
+ "label": "foo",
+ "namespace": "example",
+ }
+ ],
+ linked_kitems=[linked_kitem3],
+ user_groups=[user_group],
+ apps=[app],
+ )
expected = {
"kitems_to_link": [
@@ -173,6 +175,7 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address):
"contexts_to_add_in": [],
"contexts_to_remove_from": [],
}
+
diffs = _get_kitems_diffs(kitem_old, kitem_new)
for key, value in diffs.items():