release v0.10.0 (#144)

README.md

@@ -41,7 +41,8 @@ Below is a list of features that we would like to implement or have been request
 | Uplift to Python 3.11                                                           | 0.2.0             | Yes       |
 | Uplift Pyspark to 3.5                                                           | 0.8.0             | Yes       |
 | Allow DVE to run on Python 3.12+                                                | 0.8.0             | Yes       |
-| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes        |
+| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes       |
+| Upgrade DuckDB to v1.4                                                          | 0.10.0            | Yes       |
 | Uplift Pyspark to 4.0+                                                          | TBA               | No        |
 | Polars upgrade to v1+ | TBA | No |
 | DuckDB upgrade to v1.5+ | TBA | No |

docs/advanced_guidance/json_schemas/dataset.schema.json

@@ -10,6 +10,9 @@
         },
         "transformations": {
             "$ref": "transformations/transformations.schema.json"
+        },
+        "entity_relationships": {
+            "$ref": "entity_relationships.schema.json"
         }
     },
     "required": [

docs/advanced_guidance/json_schemas/entity_relationships.schema.json

@@ -0,0 +1,37 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "data-ingest:entity_relationships.schema.json",
+    "title": "entity_relationships",
+    "description": "Description of relationships to link normalised entities back to parent entities.",
+    "type": "object",
+    "patternProperties": {
+        "^[A-Za-z0-9_]+.$": {
+            "type": "object",
+            "properties": {
+                "parent_entity": {
+                    "type": "string"
+                },
+                "join_fields": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "mandatory": {
+                    "type": "boolean"
+                },
+                "orphaned_records_error_code": {
+                    "type": "string"
+                },
+                "orphaned_records_error_message": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "parent_entity",
+                "join_fields"
+            ],
+            "additionalProperties": false
+        }
+    }
+}

docs/user_guidance/install.md

@@ -78,11 +78,12 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru
 
 ## DVE Version Compatability Matrix
 
-| DVE Version  | Python Version | DuckDB Version | Spark Version   | Pydantic Version |
-| ------------ | -------------- | -------------- | --------------- | ---------------- |
-| >=0.9.0      | >=3.10,<3.13   | 1.1.3          | >=3.5.0,<=3.5.5 | 2.13.4           | 
-| >=0.8.0      | >=3.10,<3.13   | 1.1.3          | 3.5.2           | 1.10.19          |
-| >=0.7.2      | >=3.10,<3.12   | 1.1.*          | 3.4.*           | 1.10.16          |
-| >=0.6        | >=3.10,<3.12   | 1.1.*          | 3.4.*           | 1.10.15          |
-| >=0.2,<0.6   | >=3.10,<3.12   | 1.1.0          | 3.4.4           | 1.10.15          |
-| 0.1          | >=3.7.2,<3.8   | 1.1.0          | 3.2.1           | 1.10.15          |
+| DVE Version  | Python Version   | DuckDB Version   | Spark Version   | Pydantic Version |
+| ------------ | --------------   | ---------------- | --------------- | ---------------- |
+| >=0.10.0     | >=3.10,<1.13     | __>=1.4,<1.4.5__ | >=3.5.0,<=3.5.5 | 2.13.4           |
+| >=0.9.0      | >=3.10,<3.13     | 1.1.3            | >=3.5.0,<=3.5.5 | __2.13.4__       |
+| >=0.8.0      | >=3.10,<3.13     | __1.1.3__        | __3.5.2__       | 1.10.19          |
+| >=0.7.2      | >=3.10,<3.12     | 1.1.*            | 3.4.*           | __1.10.16__      |
+| >=0.6        | >=3.10,<3.12     | __1.1.*__        | __3.4.*__       | 1.10.15          |
+| >=0.2,<0.6   | __>=3.10,<3.12__ | 1.1.0            | 3.4.4           | 1.10.15          |
+| 0.1          | >=3.7.2,<3.8     | 1.1.0            | 3.2.1           | 1.10.15          |

pyproject.toml

@@ -7,6 +7,7 @@ authors = [
 ]
 readme = "README.md"
 classifiers = [
+    "Development Status :: 4 - Beta",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -35,7 +36,7 @@ python = ">=3.10,<3.13"  # breaking changes beyond 3.12
 boto3 = ">=1.34.162,<1.36"  # breaking change beyond 1.36
 botocore = ">=1.34.162,<1.36"  # breaking change beyond 1.36
 delta-spark = ">=3.0.0,<=3.2.0"
-duckdb = "1.1.3"  # breaking changes beyond 1.1
+duckdb = ">=1.4,<1.4.5"
 Jinja2 = "3.1.6"
 lxml = "6.1.1"
 numpy = "1.26.4"

src/dve/core_engine/backends/implementations/duckdb/rules.py

@@ -364,7 +364,7 @@ def join_header(self, entities: DuckDBEntities, *, config: HeaderJoin) -> Messag
                 ),
             )
 
-        target_schema = DDBStruct(dict(zip(target_rel.columns, target_rel.dtypes)))()
+        target_schema = DDBStruct(dict(zip(target_rel.columns, target_rel.dtypes)))()  # type: ignore  # pylint:disable=C0301
 
         joined_rel = source_rel.select(
             StarExpression(exclude=[]),