From 07d2059be0b487901a9dc015eeff5391ae55c485 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Tue, 28 Jan 2025 16:43:02 +0100
Subject: [PATCH 1/2] lint, QTY, etc
---
.gitlab-ci.yml | 2 +-
pyproject.toml | 17 ++++
stac_extension_genmeta/__init__.py | 5 +-
stac_extension_genmeta/core.py | 71 ++++++++--------
stac_extension_genmeta/schema.py | 130 ++++++++---------------------
stac_extension_genmeta/testing.py | 75 ++++++++---------
tests/extensions_test.py | 15 ++--
7 files changed, 135 insertions(+), 180 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4bb2302..4038e12 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,6 +16,6 @@ Tests:
except: [main, tags]
script:
- pip install pip --upgrade
- - pip install .
+ - pip install .[tests]
- python3 tests/extensions_test.py
diff --git a/pyproject.toml b/pyproject.toml
index c4b3d8d..0ad4e6c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,6 +28,7 @@ packages = ["stac_extension_genmeta"]
[project.optional-dependencies]
validation = ["requests", "pystac[validation]"]
+tests = ["requests"]
[tool.setuptools.dynamic]
version = { attr = "stac_extension_genmeta.__version__" }
@@ -35,3 +36,19 @@ version = { attr = "stac_extension_genmeta.__version__" }
[tool.pydocstyle]
convention = "google"
+[tool.mypy]
+plugins = [
+ "pydantic.mypy"
+]
+show_error_codes = true
+pretty = true
+exclude = ["doc", "venv", ".venv"]
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true
+warn_untyped_fields = true
+
+[tool.pylint]
+disable = "W1203,R0903,E0401,W0622,C0116,C0115,W0719"
diff --git a/stac_extension_genmeta/__init__.py b/stac_extension_genmeta/__init__.py
index eea0c50..adcf023 100644
--- a/stac_extension_genmeta/__init__.py
+++ b/stac_extension_genmeta/__init__.py
@@ -1,2 +1,5 @@
-from .core import create_extension_cls
+"""STAC extension generic metadata."""
+
+from .core import create_extension_cls # noqa
+
__version__ = "0.1.2"
diff --git a/stac_extension_genmeta/core.py b/stac_extension_genmeta/core.py
index a63c962..d9a9333 100644
--- a/stac_extension_genmeta/core.py
+++ b/stac_extension_genmeta/core.py
@@ -1,23 +1,19 @@
"""
Processing extension
"""
-from typing import Any, Generic, TypeVar, Union, cast
-from pystac.extensions.base import PropertiesExtension, \
- ExtensionManagementMixin
-import pystac
-from pydantic import BaseModel, Field
+
import re
+from typing import Any, Generic, TypeVar, Union, cast
from collections.abc import Iterable
-from .schema import generate_schema
import json
+from pystac.extensions.base import PropertiesExtension, ExtensionManagementMixin
+import pystac
+from pydantic import BaseModel
+from .schema import generate_schema
-def create_extension_cls(
- model_cls: BaseModel,
- schema_uri: str
-) -> PropertiesExtension:
- """
- This method creates a pystac extension from a pydantic model.
+def create_extension_cls(model_cls: BaseModel, schema_uri: str) -> PropertiesExtension:
+ """This method creates a pystac extension from a pydantic model.
Args:
model_cls: pydantic model class
@@ -40,34 +36,36 @@ def create_extension_cls(
class CustomExtension(
Generic[T],
PropertiesExtension,
- ExtensionManagementMixin[Union[pystac.Item, pystac.Collection]]
+ ExtensionManagementMixin[Union[pystac.Item, pystac.Collection]],
):
+ """Custom extension Class."""
+
def __init__(self, obj: T):
+ """Initializer."""
if isinstance(obj, pystac.Item):
self.properties = obj.properties
elif isinstance(obj, (pystac.Asset, pystac.Collection)):
self.properties = obj.extra_fields
else:
raise pystac.ExtensionTypeError(
- f"{model_cls.__name__} cannot be instantiated from type "
- f"{type(obj).__name__}"
+ f"{model_cls.__name__} cannot be instantiated from type {type(obj).__name__}"
)
# Try to get properties from STAC item
# If not possible, self.md is set to `None`
props = {
- key: self._get_property(info.alias, str)
+ key: self._get_property(str(info.alias), str)
for key, info in model_cls.__fields__.items()
}
props = {p: v for p, v in props.items() if v is not None}
self.md = model_cls(**props) if props else None
def __getattr__(self, item):
- # forward getattr to self.md
+ """Forward getattr to `self.md`."""
return getattr(self.md, item) if self.md else None
def apply(self, md: model_cls = None, **kwargs) -> None:
-
+ """Apply some metadata to the STAC object."""
if md is None and not kwargs:
raise ValueError("At least `md` or kwargs is required")
@@ -85,20 +83,22 @@ def create_extension_cls(
@classmethod
def get_schema_uri(cls) -> str:
+ """Return the schema URL."""
return schema_uri
@classmethod
def get_schema(cls) -> dict:
+ """Return the shema as a dict."""
return generate_schema(
model_cls=model_cls,
title=f"STAC extension from {model_cls.__name__} model",
- description=f"STAC extension based on the {model_cls.__name__} "
- "model",
- schema_uri=schema_uri
+ description=f"STAC extension based on the {model_cls.__name__} model",
+ schema_uri=schema_uri,
)
@classmethod
def print_schema(cls):
+ """Display the schema."""
print(
"\033[92mPlease copy/paste the schema below in the right place "
f"in the repository so it can be accessed from \033[94m"
@@ -107,47 +107,50 @@ def create_extension_cls(
@classmethod
def export_schema(cls, json_file):
- with open(json_file, 'w') as f:
+ """Export the schema."""
+ with open(json_file, "w", encoding="utf-8") as f:
json.dump(cls.get_schema(), f, indent=2)
@classmethod
- def ext(
- cls,
- obj: T,
- add_if_missing: bool = False
- ) -> model_cls.__name__:
+ def ext(cls, obj: T, add_if_missing: bool = False) -> model_cls.__name__:
+ """Instanciate an extension."""
if isinstance(obj, pystac.Item):
cls.ensure_has_extension(obj, add_if_missing)
return cast(CustomExtension[T], ItemCustomExtension(obj))
- elif isinstance(obj, pystac.Asset):
+ if isinstance(obj, pystac.Asset):
cls.ensure_owner_has_extension(obj, add_if_missing)
return cast(CustomExtension[T], AssetCustomExtension(obj))
- elif isinstance(obj, pystac.Collection):
+ if isinstance(obj, pystac.Collection):
cls.ensure_has_extension(obj, add_if_missing)
return cast(CustomExtension[T], CollectionCustomExtension(obj))
raise pystac.ExtensionTypeError(
- f"{model_cls.__name__} does not apply to type "
- f"{type(obj).__name__}"
+ f"{model_cls.__name__} does not apply to type {type(obj).__name__}"
)
class ItemCustomExtension(CustomExtension[pystac.Item]):
- pass
+ """Item custom extension."""
class AssetCustomExtension(CustomExtension[pystac.Asset]):
+ """Asset custom extension."""
+
asset_href: str
properties: dict[str, Any]
additional_read_properties: Iterable[dict[str, Any]] | None = None
- def __init__(self, asset: pystac.Asset):
+ def __init__(self, asset: pystac.Asset): # pylint: disable=super-init-not-called
+ """Initializer."""
self.asset_href = asset.href
self.properties = asset.extra_fields
if asset.owner and isinstance(asset.owner, pystac.Item):
self.additional_read_properties = [asset.owner.properties]
class CollectionCustomExtension(CustomExtension[pystac.Collection]):
+ """Collection custom extension."""
+
properties: dict[str, Any]
- def __init__(self, collection: pystac.Collection):
+ def __init__(self, collection: pystac.Collection): # pylint: disable=super-init-not-called
+ """Initializer."""
self.properties = collection.extra_fields
CustomExtension.__name__ = f"CustomExtensionFrom{model_cls.__name__}"
diff --git a/stac_extension_genmeta/schema.py b/stac_extension_genmeta/schema.py
index ffe1cd7..adf35c4 100644
--- a/stac_extension_genmeta/schema.py
+++ b/stac_extension_genmeta/schema.py
@@ -1,12 +1,10 @@
+"""Schema generation."""
+
from pydantic import BaseModel
-def generate_schema(
- model_cls: BaseModel,
- title: str,
- description: str,
- schema_uri: str
-) -> dict:
+def generate_schema(model_cls: BaseModel, title: str, description: str, schema_uri: str) -> dict:
+ """Generate the schema from the model."""
properties = model_cls.model_json_schema()
# prune "required"
properties.pop("required", None)
@@ -21,116 +19,60 @@ def generate_schema(
"allOf": [
{
"type": "object",
- "required": [
- "type",
- "properties",
- "assets",
- "links"
- ],
+ "required": ["type", "properties", "assets", "links"],
"properties": {
- "type": {
- "const": "Feature"
- },
- "properties": {
- "$ref": "#/definitions/fields"
- },
- "assets": {
- "$ref": "#/definitions/assets"
- },
- "links": {
- "$ref": "#/definitions/links"
- }
- }
+ "type": {"const": "Feature"},
+ "properties": {"$ref": "#/definitions/fields"},
+ "assets": {"$ref": "#/definitions/assets"},
+ "links": {"$ref": "#/definitions/links"},
+ },
},
- {
- "$ref": "#/definitions/stac_extensions"
- }
- ]
+ {"$ref": "#/definitions/stac_extensions"},
+ ],
},
{
"$comment": "This is the schema for STAC Collections.",
"allOf": [
{
"type": "object",
- "required": [
- "type"
- ],
+ "required": ["type"],
"properties": {
- "type": {
- "const": "Collection"
- },
- "assets": {
- "$ref": "#/definitions/assets"
- },
- "item_assets": {
- "$ref": "#/definitions/assets"
- },
- "links": {
- "$ref": "#/definitions/links"
- }
- }
- },
- {
- "$ref": "#/definitions/fields"
+ "type": {"const": "Collection"},
+ "assets": {"$ref": "#/definitions/assets"},
+ "item_assets": {"$ref": "#/definitions/assets"},
+ "links": {"$ref": "#/definitions/links"},
+ },
},
- {
- "$ref": "#/definitions/stac_extensions"
- }
- ]
+ {"$ref": "#/definitions/fields"},
+ {"$ref": "#/definitions/stac_extensions"},
+ ],
},
{
"$comment": "This is the schema for STAC Catalogs.",
"allOf": [
{
"type": "object",
- "required": [
- "type"
- ],
+ "required": ["type"],
"properties": {
- "type": {
- "const": "Catalog"
- },
- "links": {
- "$ref": "#/definitions/links"
- }
- }
- },
- {
- "$ref": "#/definitions/fields"
+ "type": {"const": "Catalog"},
+ "links": {"$ref": "#/definitions/links"},
+ },
},
- {
- "$ref": "#/definitions/stac_extensions"
- }
- ]
- }
+ {"$ref": "#/definitions/fields"},
+ {"$ref": "#/definitions/stac_extensions"},
+ ],
+ },
],
"definitions": {
"stac_extensions": {
"type": "object",
- "required": [
- "stac_extensions"
- ],
+ "required": ["stac_extensions"],
"properties": {
- "stac_extensions": {
- "type": "array",
- "contains": {
- "const": schema_uri
- }
- }
- }
- },
- "links": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/fields"
- }
- },
- "assets": {
- "type": "object",
- "additionalProperties": {
- "$ref": "#/definitions/fields"
- }
+ "stac_extensions": {"type": "array", "contains": {"const": schema_uri}}
+ },
},
- "fields": properties
- }
+ "links": {"type": "array", "items": {"$ref": "#/definitions/fields"}},
+ "assets": {"type": "object", "additionalProperties": {"$ref": "#/definitions/fields"}},
+ "fields": properties,
+ },
}
diff --git a/stac_extension_genmeta/testing.py b/stac_extension_genmeta/testing.py
index 5f51e2d..f0c294f 100644
--- a/stac_extension_genmeta/testing.py
+++ b/stac_extension_genmeta/testing.py
@@ -1,10 +1,15 @@
-import pystac
-from datetime import datetime
+"""Perform tests."""
+
import random
import json
+import difflib
+from datetime import datetime
+import requests
+import pystac
def create_dummy_item(date=None):
+ """Create a dummy STAC item."""
if not date:
date = datetime.now().replace(year=1999)
@@ -15,21 +20,19 @@ def create_dummy_item(date=None):
geom = {
"type": "Polygon",
"coordinates": [
- [[4.032730583418401, 43.547450099338604],
- [4.036414917971517, 43.75162726634343],
- [3.698685718905037, 43.75431706444037],
- [3.6962018175925073, 43.55012996681564],
- [4.032730583418401, 43.547450099338604]]
- ]
+ [
+ [4.032730583418401, 43.547450099338604],
+ [4.036414917971517, 43.75162726634343],
+ [3.698685718905037, 43.75431706444037],
+ [3.6962018175925073, 43.55012996681564],
+ [4.032730583418401, 43.547450099338604],
+ ]
+ ],
}
- asset = pystac.Asset(
- href="https://example.com/SP67_FR_subset_1.tif"
- )
+ asset = pystac.Asset(href="https://example.com/SP67_FR_subset_1.tif")
val = f"item_{random.uniform(10000, 80000)}"
spat_extent = pystac.SpatialExtent([[0, 0, 2, 3]])
- temp_extent = pystac.TemporalExtent(
- intervals=[(None, None)]
- )
+ temp_extent = pystac.TemporalExtent(intervals=[(None, None)])
item = pystac.Item(
id=val,
@@ -39,7 +42,7 @@ def create_dummy_item(date=None):
properties={},
assets={"ndvi": asset},
href="https://example.com/collections/collection-test3/items/{val}",
- collection="collection-test3"
+ collection="collection-test3",
)
col = pystac.Collection(
@@ -56,17 +59,16 @@ def create_dummy_item(date=None):
METHODS = ["arg", "md", "dict"]
-def basic_test(
- ext_md,
- ext_cls,
- item_test: bool = True,
- asset_test: bool = True,
- collection_test: bool = True,
- validate: bool = True
+def basic_test( # pylint: disable=too-many-arguments,too-many-positional-arguments
+ ext_md,
+ ext_cls,
+ item_test: bool = True,
+ asset_test: bool = True,
+ collection_test: bool = True,
+ validate: bool = True,
):
- print(
- f"Extension metadata model: \n{ext_md.__class__.schema_json(indent=2)}"
- )
+ """Perform some basic tests."""
+ print(f"Extension metadata model: \n{ext_md.__class__.schema_json(indent=2)}")
def apply(stac_obj, method="arg"):
"""
@@ -79,10 +81,7 @@ def basic_test(
elif method == "md":
ext.apply(md=ext_md)
elif method == "dict":
- d = {
- name: getattr(ext_md, name)
- for name in ext_md.__fields__
- }
+ d = {name: getattr(ext_md, name) for name in ext_md.__fields__}
print(f"Passing kwargs: {d}")
ext.apply(**d)
@@ -130,7 +129,7 @@ def basic_test(
"""
Test extension against collection
"""
- item, col = create_dummy_item()
+ _, col = create_dummy_item()
print_item(col)
apply(col, method)
print_item(col)
@@ -152,10 +151,11 @@ def basic_test(
def is_schema_url_synced(cls):
- import requests
+ """Check that the hosted schema is up-to-date."""
+
local_schema = cls.get_schema()
url = cls.get_schema_uri()
- remote_schema = requests.get(url).json()
+ remote_schema = requests.get(url, timeout=10).json()
print(
f"Local schema is :\n"
f"{local_schema}\n"
@@ -165,15 +165,10 @@ def is_schema_url_synced(cls):
)
if local_schema != remote_schema:
print("Schema differs:")
- import difflib
+
def _json2str(dic):
return json.dumps(dic, indent=2).split("\n")
- diff = difflib.unified_diff(
- _json2str(local_schema),
- _json2str(remote_schema)
- )
+ diff = difflib.unified_diff(_json2str(local_schema), _json2str(remote_schema))
print("\n".join(diff))
- raise ValueError(
- f"Please update the schema located in {url}"
- )
+ raise ValueError(f"Please update the schema located in {url}")
diff --git a/tests/extensions_test.py b/tests/extensions_test.py
index 85ec701..1158878 100644
--- a/tests/extensions_test.py
+++ b/tests/extensions_test.py
@@ -18,21 +18,16 @@ class MyExtensionMetadataModel(BaseModel):
name: str = Field(title="Process name", alias=f"{PREFIX}:name")
authors: List[str] = Field(title="Authors", alias=f"{PREFIX}:authors")
version: str = Field(title="Process version", alias=f"{PREFIX}:version")
- opt_field: str | None = Field(title="Some optional field", alias=f"{PREFIX}:opt_field", default=None)
+ opt_field: str | None = Field(
+ title="Some optional field", alias=f"{PREFIX}:opt_field", default=None
+ )
# Create the extension class
-MyExtension = create_extension_cls(
- model_cls=MyExtensionMetadataModel,
- schema_uri=SCHEMA_URI
-)
+MyExtension = create_extension_cls(model_cls=MyExtensionMetadataModel, schema_uri=SCHEMA_URI)
# Metadata fields
-ext_md = MyExtensionMetadataModel(
- name="test",
- authors=["michel", "denis"],
- version="alpha"
-)
+ext_md = MyExtensionMetadataModel(name="test", authors=["michel", "denis"], version="alpha")
basic_test(ext_md, MyExtension, validate=False)
--
GitLab
From 969e44644f5d15d67891e50cd91f4e716fafde79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Tue, 28 Jan 2025 17:17:15 +0100
Subject: [PATCH 2/2] lint, QTY, etc
---
pyproject.toml | 4 +---
stac_extension_genmeta/core.py | 11 +++++------
tests/extensions_test.py | 7 +++++--
3 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 0ad4e6c..5ae2498 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,9 +37,7 @@ version = { attr = "stac_extension_genmeta.__version__" }
convention = "google"
[tool.mypy]
-plugins = [
- "pydantic.mypy"
-]
+plugins = ["pydantic.mypy"]
show_error_codes = true
pretty = true
exclude = ["doc", "venv", ".venv"]
diff --git a/stac_extension_genmeta/core.py b/stac_extension_genmeta/core.py
index d9a9333..4bc5be0 100644
--- a/stac_extension_genmeta/core.py
+++ b/stac_extension_genmeta/core.py
@@ -1,9 +1,8 @@
-"""
-Processing extension
-"""
+# mypy: ignore-errors
+"""Processing extension."""
import re
-from typing import Any, Generic, TypeVar, Union, cast
+from typing import Any, Generic, TypeVar, Union, cast, Type
from collections.abc import Iterable
import json
from pystac.extensions.base import PropertiesExtension, ExtensionManagementMixin
@@ -12,7 +11,7 @@ from pydantic import BaseModel
from .schema import generate_schema
-def create_extension_cls(model_cls: BaseModel, schema_uri: str) -> PropertiesExtension:
+def create_extension_cls(model_cls: Type[BaseModel], schema_uri: str):
"""This method creates a pystac extension from a pydantic model.
Args:
@@ -58,7 +57,7 @@ def create_extension_cls(model_cls: BaseModel, schema_uri: str) -> PropertiesExt
for key, info in model_cls.__fields__.items()
}
props = {p: v for p, v in props.items() if v is not None}
- self.md = model_cls(**props) if props else None
+ self.md = model_cls.parse_obj(**props) if props else None
def __getattr__(self, item):
"""Forward getattr to `self.md`."""
diff --git a/tests/extensions_test.py b/tests/extensions_test.py
index 1158878..d2ae7d9 100644
--- a/tests/extensions_test.py
+++ b/tests/extensions_test.py
@@ -1,7 +1,9 @@
+# mypy: ignore-errors
+"""Tests."""
from stac_extension_genmeta import create_extension_cls
from stac_extension_genmeta.testing import basic_test
from pydantic import BaseModel, Field, ConfigDict
-from typing import List
+from typing import List, Type
# Extension parameters
SCHEMA_URI: str = "https://example.com/image-process/v1.0.0/schema.json"
@@ -24,7 +26,8 @@ class MyExtensionMetadataModel(BaseModel):
# Create the extension class
-MyExtension = create_extension_cls(model_cls=MyExtensionMetadataModel, schema_uri=SCHEMA_URI)
+model_cls: Type[BaseModel] = MyExtensionMetadataModel
+MyExtension = create_extension_cls(model_cls=model_cls, schema_uri=SCHEMA_URI)
# Metadata fields
ext_md = MyExtensionMetadataModel(name="test", authors=["michel", "denis"], version="alpha")
--
GitLab