From 7364524a3c20e3bdc19439c1cddba0b5ce111304 Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Mon, 23 Feb 2026 14:31:46 +0200 Subject: [PATCH 1/9] Add import mappings for alternative descriptions Re spine-tools/Spine-Toolbox#1892 --- spinedb_api/import_mapping/import_mapping.py | 25 ++++++++++++++-- .../import_mapping/import_mapping_compat.py | 5 ---- tests/import_mapping/test_generator.py | 29 +++++++++++++++++++ 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/spinedb_api/import_mapping/import_mapping.py b/spinedb_api/import_mapping/import_mapping.py index 8c62c406..94dc7edf 100644 --- a/spinedb_api/import_mapping/import_mapping.py +++ b/spinedb_api/import_mapping/import_mapping.py @@ -906,7 +906,6 @@ class ParameterValueListValueMapping(ImportMapping): """Maps parameter value list values. Cannot be used as the topmost mapping; must have a :class:`ParameterValueListMapping` as parent. - """ MAP_TYPE = "ParameterValueListValue" @@ -932,6 +931,24 @@ def _import_row(self, source_data, state, mapped_data): mapped_data.setdefault("alternatives", set()).add(alternative) +class AlternativeDescriptionMapping(ImportMapping): + """Maps alternative descriptions. + + Cannot be used as the topmost mapping; must have a :class:`AlternativeMapping` as parent. + """ + + MAP_TYPE = "AlternativeDescription" + ignorable = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + alternative = state[ImportKey.ALTERNATIVE_NAME] + alternative_data = mapped_data["alternatives"] + alternative_data.discard(alternative) + alternative_data.add((alternative, description)) + + class ScenarioMapping(ImportMapping): """Maps scenarios. @@ -1012,13 +1029,14 @@ def _default_entity_class_mapping(): return root_mapping -def _default_alternative_mapping(): +def _default_alternative_mapping() -> AlternativeMapping: """Creates default alternative mappings. Returns: - AlternativeMapping: root mapping + root mapping """ root_mapping = AlternativeMapping(Position.hidden) + root_mapping.child = AlternativeDescriptionMapping(Position.hidden) return root_mapping @@ -1134,6 +1152,7 @@ def from_dict(serialized): MetadataNameMapping, MetadataValueMapping, AlternativeMapping, + AlternativeDescriptionMapping, ScenarioMapping, ScenarioAlternativeMapping, ScenarioBeforeAlternativeMapping, diff --git a/spinedb_api/import_mapping/import_mapping_compat.py b/spinedb_api/import_mapping/import_mapping_compat.py index c8afaaed..d4350b6d 100644 --- a/spinedb_api/import_mapping/import_mapping_compat.py +++ b/spinedb_api/import_mapping/import_mapping_compat.py @@ -20,8 +20,6 @@ EntityClassMapping, EntityGroupMapping, EntityMapping, - EntityMetadataNameMapping, - EntityMetadataValueMapping, ExpandedParameterDefaultValueMapping, ExpandedParameterValueMapping, IndexNameMapping, @@ -33,8 +31,6 @@ ParameterValueListMapping, ParameterValueListValueMapping, ParameterValueMapping, - ParameterValueMetadataNameMapping, - ParameterValueMetadataValueMapping, ParameterValueTypeMapping, Position, ScenarioAlternativeMapping, @@ -132,7 +128,6 @@ def _scenario_alternative_mapping_from_dict(map_dict): def _object_class_mapping_from_dict(map_dict): name = map_dict.get("name") entities = map_dict.get("objects", map_dict.get("object")) - object_metadata = map_dict.get("object_metadata", None) parameters = map_dict.get("parameters") skip_columns = map_dict.get("skip_columns", []) read_start_row = map_dict.get("read_start_row", 0) diff --git a/tests/import_mapping/test_generator.py b/tests/import_mapping/test_generator.py index ab88c118..2e49c406 100644 --- a/tests/import_mapping/test_generator.py +++ b/tests/import_mapping/test_generator.py @@ -14,7 +14,9 @@ import unittest from spinedb_api import Array, DateTime, Duration, Map from spinedb_api.import_mapping.generator import get_mapped_data +from spinedb_api.import_mapping.import_mapping import default_import_mapping from spinedb_api.import_mapping.type_conversion import value_to_convert_spec +from spinedb_api.mapping import to_dict, unflatten class TestGetMappedData(unittest.TestCase): @@ -1175,3 +1177,30 @@ def test_import_parameter_value_metadata(self): ], }, ) + + def test_import_alternatives_with_descriptions(self): + header = ["Alternative", "Description"] + data_source = iter( + [ + ["alt1", "First alternative."], + ["alt2", None], + ["duplicate", "Overridden description."], + ["duplicate", "Overriding description."], + ] + ) + flattened = default_import_mapping("Alternative").flatten() + flattened[0].position = 0 + flattened[1].position = 1 + mapped_data, errors = get_mapped_data(data_source, [to_dict(unflatten(flattened))], header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "alternatives": { + ("alt1", "First alternative."), + "alt2", + ("duplicate", "Overridden description."), + ("duplicate", "Overriding description."), + }, + }, + ) From 53317369be30cd3aeb043822f8d8a171dd2f00c3 Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Mon, 23 Feb 2026 15:34:45 +0200 Subject: [PATCH 2/9] Add import mappings for scenario descriptions Re spine-tools/Spine-Toolbox#1892 --- spinedb_api/import_mapping/import_mapping.py | 31 +++++++++++++---- tests/import_mapping/test_generator.py | 35 ++++++++++++++++++-- tests/import_mapping/test_import_mapping.py | 28 +++++++++------- 3 files changed, 74 insertions(+), 20 deletions(-) diff --git a/spinedb_api/import_mapping/import_mapping.py b/spinedb_api/import_mapping/import_mapping.py index 94dc7edf..087b398c 100644 --- a/spinedb_api/import_mapping/import_mapping.py +++ b/spinedb_api/import_mapping/import_mapping.py @@ -960,8 +960,7 @@ class ScenarioMapping(ImportMapping): def _import_row(self, source_data, state, mapped_data): scenario = str(source_data) state[ImportKey.SCENARIO_NAME] = scenario - if self._child is None: - mapped_data.setdefault("scenarios", set()).add((scenario,)) + mapped_data.setdefault("scenarios", set()).add((scenario,)) class ScenarioAlternativeMapping(ImportMapping): @@ -995,6 +994,24 @@ def _import_row(self, source_data, state, mapped_data): scen_alt.append(alternative) +class ScenarioDescriptionMapping(ImportMapping): + """Maps scenario descriptions. + + Cannot be used as the topmost mapping; must have a :class:`ScenarioMapping` as parent. + """ + + MAP_TYPE = "ScenarioDescription" + ignorable: ClassVar[bool] = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + scenario = state[ImportKey.SCENARIO_NAME] + scenario_data = mapped_data["scenarios"] + scenario_data.discard((scenario,)) + scenario_data.add((scenario, description)) + + def default_import_mapping(map_type: str) -> ImportMapping: """Creates default mappings for given map type. @@ -1040,21 +1057,22 @@ def _default_alternative_mapping() -> AlternativeMapping: return root_mapping -def _default_scenario_mapping(): +def _default_scenario_mapping() -> ScenarioMapping: """Creates default scenario mappings. Returns: - ScenarioMapping: root mapping + root mapping """ root_mapping = ScenarioMapping(Position.hidden) + root_mapping.child = ScenarioDescriptionMapping(Position.hidden) return root_mapping -def _default_scenario_alternative_mapping(): +def _default_scenario_alternative_mapping() -> ScenarioMapping: """Creates default scenario alternative mappings. Returns: - ScenarioAlternativeMapping: root mapping + root mapping """ root_mapping = ScenarioMapping(Position.hidden) root_mapping.child = ScenarioAlternativeMapping(Position.hidden) @@ -1156,6 +1174,7 @@ def from_dict(serialized): ScenarioMapping, ScenarioAlternativeMapping, ScenarioBeforeAlternativeMapping, + ScenarioDescriptionMapping, ) } legacy_mappings = { diff --git a/tests/import_mapping/test_generator.py b/tests/import_mapping/test_generator.py index 2e49c406..f809abdd 100644 --- a/tests/import_mapping/test_generator.py +++ b/tests/import_mapping/test_generator.py @@ -976,7 +976,10 @@ def test_skip_first_row_when_importing_pivoted_data(self): self.assertEqual(errors, []) self.assertEqual( mapped_data, - {"scenario_alternatives": [["Scenario1", "Base"], ["Scenario1", "fixed_prices"]]}, + { + "scenarios": {("Scenario1",)}, + "scenario_alternatives": [["Scenario1", "Base"], ["Scenario1", "fixed_prices"]], + }, ) def test_leaf_mapping_with_position_on_row_is_still_considered_as_pivoted(self): @@ -1001,13 +1004,14 @@ def test_leaf_mapping_with_position_on_row_is_still_considered_as_pivoted(self): self.assertEqual( mapped_data, { + "scenarios": {("Scenario1",), ("Scenario2",)}, "scenario_alternatives": [ ["Scenario1", "Base"], ["Scenario1", "alt1"], ["Scenario2", "Base"], ["Scenario2", "alt1"], ["Scenario2", "alt2"], - ] + ], }, ) @@ -1204,3 +1208,30 @@ def test_import_alternatives_with_descriptions(self): }, }, ) + + def test_import_scenarios_with_descriptions(self): + header = ["Scenario", "Description"] + data_source = iter( + [ + ["scen1", "First scenario."], + ["scen2", None], + ["duplicate", "Possible description no. 1."], + ["duplicate", "Possible description no. 2."], + ] + ) + flattened = default_import_mapping("Scenario").flatten() + flattened[0].position = 0 + flattened[1].position = 1 + mapped_data, errors = get_mapped_data(data_source, [to_dict(unflatten(flattened))], header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "scenarios": { + ("scen1", "First scenario."), + ("scen2",), + ("duplicate", "Possible description no. 1."), + ("duplicate", "Possible description no. 2."), + }, + }, + ) diff --git a/tests/import_mapping/test_import_mapping.py b/tests/import_mapping/test_import_mapping.py index 0bea2a4c..2238760b 100644 --- a/tests/import_mapping/test_import_mapping.py +++ b/tests/import_mapping/test_import_mapping.py @@ -1697,12 +1697,14 @@ def test_read_scenario_alternative(self): "before_alternative_name": 2, } out, errors = get_mapped_data(data, [mapping], data_header) - expected = {} - expected["scenario_alternatives"] = [ - ["scenario_A", "alternative1", "second_alternative"], - ["scenario_A", "second_alternative", "last_one"], - ["scenario_B", "last_one", ""], - ] + expected = { + "scenarios": {("scenario_A",), ("scenario_B",)}, + "scenario_alternatives": [ + ["scenario_A", "alternative1", "second_alternative"], + ["scenario_A", "second_alternative", "last_one"], + ["scenario_B", "last_one", ""], + ], + } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1711,12 +1713,14 @@ def test_pivoted_scenario_alternative(self): data = iter(input_data) mappings = [{"map_type": "Scenario", "position": -1}, {"map_type": "ScenarioAlternative", "position": "hidden"}] out, errors = get_mapped_data(data, [mappings]) - expected = {} - expected["scenario_alternatives"] = [ - ["scenario_A", "first_alternative"], - ["scenario_A", "second_alternative"], - ["scenario_B", "Base"], - ] + expected = { + "scenarios": {("scenario_A",), ("scenario_B",)}, + "scenario_alternatives": [ + ["scenario_A", "first_alternative"], + ["scenario_A", "second_alternative"], + ["scenario_B", "Base"], + ], + } self.assertFalse(errors) self.assertEqual(out, expected) From 599c07b15a1fe7cda9b3cb6390207576d817f9d2 Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Wed, 25 Feb 2026 08:43:41 +0200 Subject: [PATCH 3/9] Add import mappings for entity class descriptions Changed the logic on how entity (class) mappings work since adding descriptions to the previous system turned out to be quite hacky. We now collect the imported data into 'records' (EntityClassRecord, EntityRecord) which we process into import data for import_functions at the end of get_mapped_data(). Also, removed the raise KeyError()/raise KeyFix() system that didn't make sense to do on every single imported row. We should have some dedicated machinery that validates the mappings without affecting the actual import process instead. In any case, the changes seem to improve the performance by a lot: unit tests on my system execute in 2:45 with these changes while before this commit, they took 3:30. Also a real-life Importer execution time is down to 5s from 9s. Re spine-tools/Spine-Toolbox#1892 --- spinedb_api/import_mapping/generator.py | 33 ++- spinedb_api/import_mapping/import_mapping.py | 197 ++++++++-------- tests/import_mapping/test_generator.py | 160 ++++++++----- tests/import_mapping/test_import_mapping.py | 236 ++++++++++--------- tests/spine_io/importers/test_reader.py | 6 +- tests/spine_io/test_excel_integration.py | 10 +- 6 files changed, 352 insertions(+), 290 deletions(-) diff --git a/spinedb_api/import_mapping/generator.py b/spinedb_api/import_mapping/generator.py index cd6dc8ff..3dde0ad5 100644 --- a/spinedb_api/import_mapping/generator.py +++ b/spinedb_api/import_mapping/generator.py @@ -16,7 +16,6 @@ """ from collections.abc import Callable from copy import deepcopy -from operator import itemgetter from typing import Any, Optional from ..exception import ParameterValueFormatError from ..helpers import string_to_bool @@ -295,23 +294,31 @@ def _unpivot_rows( return unpivoted_rows, pivoted_pos, non_pivoted_pos, unpivoted_column_pos -def _make_entity_classes(mapped_data): - rows = mapped_data.get("entity_classes") - if rows is None: +def _make_entity_classes(mapped_data: dict) -> None: + try: + rows = mapped_data.pop("entity_classes") + except KeyError: return - rows = [(class_name, tuple(dimension_names)) for class_name, dimension_names in rows.items()] - rows.sort(key=itemgetter(1)) - mapped_data["entity_classes"] = final_rows = [] - for class_name, dimension_names in rows: - row = (class_name, tuple(dimension_names)) if dimension_names else (class_name,) - final_rows.append(row) + final_rows = [] + for name, record in rows.items(): + item = [name, record.dimensions] + if record.description: + item.append(record.description) + final_rows.append(item) + if final_rows: + mapped_data["entity_classes"] = final_rows def _make_entities(mapped_data): - rows = mapped_data.get("entities") - if rows is None: + try: + rows = mapped_data.pop("entities") + except KeyError: return - mapped_data["entities"] = list(rows) + final_rows = [] + for (class_name, name), record in rows.items(): + final_rows.append((class_name, name if not record.elements else record.elements)) + if final_rows: + mapped_data["entities"] = final_rows def _make_entity_alternatives(mapped_data, errors): diff --git a/spinedb_api/import_mapping/import_mapping.py b/spinedb_api/import_mapping/import_mapping.py index 087b398c..2549a2a8 100644 --- a/spinedb_api/import_mapping/import_mapping.py +++ b/spinedb_api/import_mapping/import_mapping.py @@ -11,17 +11,18 @@ ###################################################################################################################### """Contains import mappings for database items such as entities, entity classes and parameter values.""" from collections.abc import Iterable +from dataclasses import dataclass, field from enum import Enum, auto, unique -from typing import Any, ClassVar +from typing import Any, ClassVar, TypeAlias from spinedb_api.exception import InvalidMapping, InvalidMappingComponent from spinedb_api.mapping import Mapping, Position, is_pivoted, parse_fixed_position_value, unflatten @unique class ImportKey(Enum): - DIMENSION_COUNT = auto() ENTITY_CLASS_NAME = auto() ENTITY_NAME = auto() + ELEMENT_NAMES = auto() GROUP_NAME = auto() MEMBER_NAME = auto() METADATA_NAME = auto() @@ -35,8 +36,6 @@ class ImportKey(Enum): PARAMETER_VALUE_INDEXES = auto() PARAMETER_VALUE_METADATA_NAME = auto() PARAMETER_VALUE_METADATA_VALUE = auto() - DIMENSION_NAMES = auto() - ELEMENT_NAMES = auto() ALTERNATIVE_NAME = auto() SCENARIO_NAME = auto() SCENARIO_ALTERNATIVE = auto() @@ -59,8 +58,6 @@ def __str__(self): self.PARAMETER_VALUE_INDEXES.value: "Parameter indexes", self.PARAMETER_VALUE_METADATA_NAME.value: "Metadata names", self.PARAMETER_VALUE_METADATA_VALUE.value: "Metadata values", - self.DIMENSION_NAMES.value: "Dimension names", - self.ELEMENT_NAMES.value: "Element names", self.PARAMETER_VALUE_LIST_NAME.value: "Parameter value lists", self.SCENARIO_NAME.value: "Scenario names", self.SCENARIO_ALTERNATIVE.value: "Alternative names", @@ -72,8 +69,8 @@ def __str__(self): return super().__str__() -class KeyFix(Exception): - """Opposite of KeyError""" +State: TypeAlias = dict[ImportKey, Any] +SemiMappedData: TypeAlias = dict[str, Any] def check_validity(root_mapping): @@ -302,12 +299,6 @@ def import_row(self, source_row, state, mapped_data, errors=None): msg = f"Required key '{key}' is invalid" error = InvalidMappingComponent(msg, self.rank, key) errors.append(error) - except KeyFix as fix: - indexes = set() - for key in fix.args: - indexes |= {k for k, err in enumerate(errors) if err.key == key} - for k in sorted(indexes, reverse=True): - errors.pop(k) if self.child is not None: self.child.import_row(source_row, state, mapped_data, errors=errors) @@ -406,6 +397,12 @@ def reconstruct(cls, position, value, skip_columns, read_start_row, filter_re, m return mapping +@dataclass +class EntityClassRecord: + dimensions: list[str] = field(default_factory=list) + description: str | None = None + + class EntityClassMapping(ImportMapping): """Maps entity classes. @@ -415,14 +412,30 @@ class EntityClassMapping(ImportMapping): MAP_TYPE = "EntityClass" def _import_row(self, source_data, state, mapped_data): - dim_count = len([m for m in self.flatten() if isinstance(m, DimensionMapping)]) - state[ImportKey.DIMENSION_COUNT] = dim_count entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] = str(source_data) - dimension_names = state[ImportKey.DIMENSION_NAMES] = [] entity_classes = mapped_data.setdefault("entity_classes", {}) - entity_classes[entity_class_name] = dimension_names - if dim_count: - raise KeyError(ImportKey.DIMENSION_NAMES) + entity_classes[entity_class_name] = EntityClassRecord() + + +class EntityClassDescriptionMapping(ImportMapping): + """Maps entity class descriptions. + + Cannot be used as the topmost mapping; one of the parents must be :class:`EntityClassMapping`. + """ + + MAP_TYPE = "EntityClassDescription" + ignorable = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + mapped_data["entity_classes"][entity_class_name].description = description + + +@dataclass +class EntityRecord: + elements: list[str] = field(default_factory=list) class EntityMapping(ImportMapping): @@ -433,18 +446,12 @@ class EntityMapping(ImportMapping): MAP_TYPE = "Entity" - def import_row(self, source_row, state, mapped_data, errors=None): - state[ImportKey.ELEMENT_NAMES] = () - super().import_row(source_row, state, mapped_data, errors=errors) - def _import_row(self, source_data, state, mapped_data): - if state[ImportKey.DIMENSION_COUNT]: + if self.position == Position.hidden and isinstance(self._child, ElementMapping): return entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] entity_name = state[ImportKey.ENTITY_NAME] = str(source_data) - if isinstance(self.child, EntityGroupMapping): - raise KeyError(ImportKey.MEMBER_NAME) - mapped_data.setdefault("entities", {})[entity_class_name, entity_name] = None + mapped_data.setdefault("entities", {})[entity_class_name, entity_name] = EntityRecord() class EntityMetadataNameMapping(ImportMapping): @@ -468,10 +475,7 @@ class EntityMetadataValueMapping(ImportMapping): def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - if state[ImportKey.DIMENSION_COUNT]: - entity_byname = state[ImportKey.ELEMENT_NAMES] - else: - entity_byname = (state[ImportKey.ENTITY_NAME],) + entity_byname = _byname_from_mapped_data(entity_class_name, state, mapped_data) metadata_name = state[ImportKey.ENTITY_METADATA_NAME] metadata_value = state[ImportKey.ENTITY_METADATA_VALUE] = source_data mapped_data.setdefault("entity_metadata", {})[ @@ -489,16 +493,16 @@ class EntityGroupMapping(ImportEntitiesMixin, ImportMapping): def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - group_name = state.get(ImportKey.ENTITY_NAME) - if group_name is None: - raise KeyError(ImportKey.GROUP_NAME) + group_name = state[ImportKey.ENTITY_NAME] member_name = str(source_data) mapped_data.setdefault("entity_groups", set()).add((entity_class_name, group_name, member_name)) if self.import_entities: - entities = mapped_data.setdefault("entities", {}) - entities[entity_class_name, group_name] = None - entities[entity_class_name, member_name] = None - raise KeyFix(ImportKey.MEMBER_NAME) + mapped_data["entities"][entity_class_name, member_name] = EntityRecord() + else: + try: + del mapped_data["entities"][entity_class_name, group_name] + except KeyError: + pass class EntityAlternativeActivityMapping(ImportMapping): @@ -512,13 +516,10 @@ class EntityAlternativeActivityMapping(ImportMapping): ignorable = True def _import_row(self, source_data, state, mapped_data): - if source_data is None or source_data == "": + if source_data == "": return entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - if state[ImportKey.DIMENSION_COUNT]: - entity_byname = state[ImportKey.ELEMENT_NAMES] - else: - entity_byname = (state[ImportKey.ENTITY_NAME],) + entity_byname = _byname_from_mapped_data(entity_class_name, state, mapped_data) alternative_name = state[ImportKey.ALTERNATIVE_NAME] mapped_data.setdefault("entity_alternatives", {})[ entity_class_name, entity_byname, alternative_name, source_data @@ -534,42 +535,48 @@ class DimensionMapping(ImportMapping): MAP_TYPE = "Dimension" def _import_row(self, source_data, state, mapped_data): - if ImportKey.ENTITY_CLASS_NAME not in state: - raise KeyError(ImportKey.ENTITY_CLASS_NAME) - dimension_names = state[ImportKey.DIMENSION_NAMES] - if len(dimension_names) == state[ImportKey.DIMENSION_COUNT]: - return dimension_name = str(source_data) - dimension_names.append(dimension_name) - if len(dimension_names) == state[ImportKey.DIMENSION_COUNT]: - raise KeyFix(ImportKey.DIMENSION_NAMES) + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + mapped_data["entity_classes"][entity_class_name].dimensions.append(dimension_name) class ElementMapping(ImportEntitiesMixin, ImportMapping): """Maps elements. - Cannot be used as the topmost mapping; must have :class:`EntityClassMapping` and :class:`EntityMapping` + Cannot be used as the topmost mapping; must have :class:`DimensionMapping` and :class:`EntityMapping` as parents. """ MAP_TYPE = "Element" def _import_row(self, source_data, state, mapped_data): - entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - dimension_names = state[ImportKey.DIMENSION_NAMES] - if len(dimension_names) != state[ImportKey.DIMENSION_COUNT]: - raise KeyError(ImportKey.DIMENSION_NAMES) element_name = str(source_data) - element_names = state[ImportKey.ELEMENT_NAMES] = state[ImportKey.ELEMENT_NAMES] + (element_name,) + if isinstance(self._child, ElementMapping): + element_names = state.setdefault(ImportKey.ELEMENT_NAMES, []) + element_names.append(element_name) + return + element_names = state.pop(ImportKey.ELEMENT_NAMES, []) + element_names.append(element_name) + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + if ImportKey.ENTITY_NAME in state: + entity_name = state[ImportKey.ENTITY_NAME] + record = mapped_data["entities"][entity_class_name, entity_name] + if all(name == existing_name for name, existing_name in zip(element_names, record.elements)): + return + del state[ImportKey.ENTITY_NAME] + record = EntityRecord(element_names) + byname = tuple(record.elements) + mapped_data.setdefault("entities", {})[entity_class_name, byname] = record + state[ImportKey.ENTITY_NAME] = byname if self.import_entities: - k = len(element_names) - 1 - dimension_name = dimension_names[k] - mapped_data.setdefault("entity_classes", {}).update({dimension_name: ()}) - mapped_data.setdefault("entities", {})[dimension_name, element_name] = None - if len(element_names) == state[ImportKey.DIMENSION_COUNT]: - mapped_data.setdefault("entities", {})[entity_class_name, tuple(element_names)] = None - raise KeyFix(ImportKey.ELEMENT_NAMES) - raise KeyError(ImportKey.ELEMENT_NAMES) + mapped_entities = mapped_data.setdefault("entities", {}) + mapped_classes = mapped_data["entity_classes"] + class_record = mapped_classes[entity_class_name] + for element_name, dimension_name in zip(element_names, class_record.dimensions): + if dimension_name not in mapped_classes: + mapped_classes[dimension_name] = EntityClassRecord() + if (dimension_name, element_name) not in mapped_entities: + mapped_entities[dimension_name, element_name] = EntityRecord() class MetadataNameMapping(ImportMapping): @@ -682,12 +689,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._id = None - def _value_key(self, state): + @staticmethod + def _value_key(state: State, mapped_data: SemiMappedData) -> tuple: raise NotImplementedError() def _import_row(self, source_data, state, mapped_data): values = state[self._STATE_KEY] - value = values[self._value_key(state)] + value = values[self._value_key(state, mapped_data)] if self._id is None: self._id = 0 current = self @@ -709,7 +717,8 @@ class DefaultValueIndexNameMapping(IndexNameMappingBase): MAP_TYPE = "DefaultValueIndexName" _STATE_KEY = ImportKey.PARAMETER_DEFAULT_VALUES - def _value_key(self, state): + @staticmethod + def _value_key(state, mapped_data): return _default_value_key(state) @@ -766,7 +775,7 @@ def _import_row(self, source_data, state, mapped_data): value = source_data if value == "": return - entity_class_name, entity_byname, parameter_name, alternative_name = _parameter_value_key(state) + entity_class_name, entity_byname, parameter_name, alternative_name = _parameter_value_key(state, mapped_data) parameter_value = [entity_class_name, entity_byname, parameter_name, value] if alternative_name is not None: parameter_value.append(alternative_name) @@ -780,7 +789,7 @@ def _import_row(self, source_data, state, mapped_data): if ImportKey.PARAMETER_NAME not in state: # Don't catch errors here, this one's invisible return - key = _parameter_value_key(state) + key = _parameter_value_key(state, mapped_data) values = state.setdefault(ImportKey.PARAMETER_VALUES, {}) if key in values: return @@ -819,10 +828,7 @@ class ParameterValueMetadataValueMapping(ImportMapping): def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] - if state[ImportKey.DIMENSION_COUNT]: - entity_byname = state[ImportKey.ELEMENT_NAMES] - else: - entity_byname = (state[ImportKey.ENTITY_NAME],) + entity_byname = _byname_from_mapped_data(entity_class_name, state, mapped_data) parameter_name = state[ImportKey.PARAMETER_NAME] alternative_name = state[ImportKey.ALTERNATIVE_NAME] metadata_name = state[ImportKey.PARAMETER_VALUE_METADATA_NAME] @@ -856,8 +862,9 @@ class IndexNameMapping(IndexNameMappingBase): MAP_TYPE = "IndexName" _STATE_KEY = ImportKey.PARAMETER_VALUES - def _value_key(self, state): - return _parameter_value_key(state) + @staticmethod + def _value_key(state, mapped_data): + return _parameter_value_key(state, mapped_data) class ExpandedParameterValueMapping(ImportMapping): @@ -874,7 +881,7 @@ class ExpandedParameterValueMapping(ImportMapping): def _import_row(self, source_data, state, mapped_data): values = state.setdefault(ImportKey.PARAMETER_VALUES, {}) - value = values[_parameter_value_key(state)] + value = values[_parameter_value_key(state, mapped_data)] data = value.setdefault("data", []) if value["type"] == "array": data.append(source_data) @@ -1035,14 +1042,15 @@ def default_import_mapping(map_type: str) -> ImportMapping: return make_root_mapping() -def _default_entity_class_mapping(): +def _default_entity_class_mapping() -> EntityClassMapping: """Creates default entity class mappings. Returns: - EntityClassMapping: root mapping + root mapping """ root_mapping = EntityClassMapping(Position.hidden) - root_mapping.child = EntityMapping(Position.hidden) + description_mapping = root_mapping.child = EntityClassDescriptionMapping(Position.hidden) + description_mapping.child = EntityMapping(Position.hidden) return root_mapping @@ -1144,6 +1152,7 @@ def from_dict(serialized): klass.MAP_TYPE: klass for klass in ( EntityClassMapping, + EntityClassDescriptionMapping, EntityMapping, EntityMetadataNameMapping, EntityMetadataValueMapping, @@ -1220,35 +1229,35 @@ def from_dict(serialized): return unflatten(flattened) -def _parameter_value_key(state): +def _parameter_value_key(state: State, mapped_data: SemiMappedData) -> tuple[str, tuple[str, ...], str, str]: """Creates parameter value's key from current state. Args: - state (dict): import state + state: import state Returns: - tuple of str: class name, entity byname, parameter name, and alternative name + class name, entity byname, parameter name, and alternative name """ entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) - if state.get(ImportKey.DIMENSION_COUNT): - element_names = state[ImportKey.ELEMENT_NAMES] - if len(element_names) != state[ImportKey.DIMENSION_COUNT]: - raise KeyError(ImportKey.ELEMENT_NAMES) - entity_byname = element_names - else: - entity_byname = state[ImportKey.ENTITY_NAME] + entity_byname = _byname_from_mapped_data(entity_class_name, state, mapped_data) parameter_name = state[ImportKey.PARAMETER_NAME] alternative_name = state.get(ImportKey.ALTERNATIVE_NAME) return entity_class_name, entity_byname, parameter_name, alternative_name -def _default_value_key(state): +def _default_value_key(state: State) -> tuple[str, str]: """Creates parameter default value's key from current state. Args: - state (dict): import state + state: import state Returns: - tuple of str: class name and parameter name + class name and parameter name """ return state[ImportKey.ENTITY_CLASS_NAME], state[ImportKey.PARAMETER_NAME] + + +def _byname_from_mapped_data(entity_class_name: str, state: State, mapped_data: SemiMappedData) -> tuple[str, ...]: + entity_name = state[ImportKey.ENTITY_NAME] + entity_record = mapped_data["entities"][entity_class_name, entity_name] + return tuple(entity_record.elements) if entity_record.elements else (entity_name,) diff --git a/tests/import_mapping/test_generator.py b/tests/import_mapping/test_generator.py index f809abdd..e92c9a9f 100644 --- a/tests/import_mapping/test_generator.py +++ b/tests/import_mapping/test_generator.py @@ -69,8 +69,8 @@ def test_returns_appropriate_error_if_last_row_is_empty(self): mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], - "parameter_values": [["Object", "data", "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], + "entity_classes": [["Object", []]], + "parameter_values": [["Object", ("data",), "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], "parameter_definitions": [("Object", "Parameter")], "entities": [("Object", "data")], }, @@ -101,8 +101,8 @@ def test_convert_functions_get_expanded_over_last_defined_column_in_pivoted_data mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], - "parameter_values": [["Object", "data", "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], + "entity_classes": [["Object", []]], + "parameter_values": [["Object", ("data",), "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], "parameter_definitions": [("Object", "Parameter")], "entities": [("Object", "data")], }, @@ -132,8 +132,8 @@ def test_read_start_row_skips_rows_in_pivoted_data(self): self.assertEqual( mapped_data, { - "entity_classes": [("klass",)], - "parameter_values": [["klass", "kloss", "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], + "entity_classes": [["klass", []]], + "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], "parameter_definitions": [("klass", "Parameter_2")], "entities": [("klass", "kloss")], }, @@ -186,7 +186,7 @@ def test_map_without_values_is_ignored_and_not_interpreted_as_null(self): mapped_data, { "alternatives": {"base"}, - "entity_classes": [("o",)], + "entity_classes": [["o", []]], "parameter_definitions": [("o", "parameter_name")], "parameter_values": [], "entities": [("o", "o1")], @@ -222,15 +222,15 @@ def test_import_object_works_with_multiple_relationship_object_imports(self): mapped_data, { "alternatives": {"base"}, - "entity_classes": [("o",), ("q",), ("o_to_q", ("o", "q"))], + "entity_classes": [["o_to_q", ["o", "q"]], ["o", []], ["q", []]], "entities": [ + ("o_to_q", ["o1", "q1"]), ("o", "o1"), ("q", "q1"), - ("o_to_q", ("o1", "q1")), + ("o_to_q", ["o2", "q2"]), ("o", "o2"), ("q", "q2"), - ("o_to_q", ("o2", "q2")), - ("o_to_q", ("o1", "q2")), + ("o_to_q", ["o1", "q2"]), ], "parameter_definitions": [("o_to_q", "param")], "parameter_values": [ @@ -266,8 +266,8 @@ def test_default_convert_function_in_column_convert_functions(self): self.assertEqual( mapped_data, { - "entity_classes": [("klass",)], - "parameter_values": [["klass", "kloss", "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], + "entity_classes": [["klass", []]], + "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], "parameter_definitions": [("klass", "Parameter_2")], "entities": [("klass", "kloss")], }, @@ -294,8 +294,8 @@ def test_identity_function_is_used_as_convert_function_when_no_convert_functions self.assertEqual( mapped_data, { - "entity_classes": [("klass",)], - "parameter_values": [["klass", "kloss", "Parameter_2", Map(["T1", "T2"], ["2.3", "23.0"])]], + "entity_classes": [["klass", []]], + "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], ["2.3", "23.0"])]], "parameter_definitions": [("klass", "Parameter_2")], "entities": [("klass", "kloss")], }, @@ -324,8 +324,8 @@ def test_last_convert_function_gets_used_as_default_convert_function_when_no_def self.assertEqual( mapped_data, { - "entity_classes": [("klass",)], - "parameter_values": [["klass", "kloss", "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], + "entity_classes": [["klass", []]], + "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], "parameter_definitions": [("klass", "Parameter_2")], "entities": [("klass", "kloss")], }, @@ -357,10 +357,10 @@ def test_array_parameters_get_imported_correctly_when_objects_are_in_header(self mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("class",)], + "entity_classes": [["class", []]], "parameter_values": [ - ["class", "object_1", "param", Array([-1.1, 1.1]), "Base"], - ["class", "object_2", "param", Array([2.3, -2.3]), "Base"], + ["class", ("object_1",), "param", Array([-1.1, 1.1]), "Base"], + ["class", ("object_2",), "param", Array([2.3, -2.3]), "Base"], ], "parameter_definitions": [("class", "param")], "entities": [("class", "object_1"), ("class", "object_2")], @@ -393,10 +393,10 @@ def test_arrays_get_imported_correctly_when_objects_are_in_header_and_alternativ mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Gadget",)], + "entity_classes": [["Gadget", []]], "parameter_values": [ - ["Gadget", "object_1", "data", Array([-1.1, 1.1]), "Base"], - ["Gadget", "object_2", "data", Array([2.3, -2.3]), "Base"], + ["Gadget", ("object_1",), "data", Array([-1.1, 1.1]), "Base"], + ["Gadget", ("object_2",), "data", Array([2.3, -2.3]), "Base"], ], "parameter_definitions": [("Gadget", "data")], "entities": [("Gadget", "object_1"), ("Gadget", "object_2")], @@ -428,12 +428,12 @@ def test_header_position_is_ignored_in_last_mapping_if_other_mappings_are_in_hea mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Data",)], + "entity_classes": [["Data", []]], "parameter_values": [ - ["Data", "d1", "parameter1", 1.1, "Base"], - ["Data", "d1", "parameter2", -2.3, "Base"], - ["Data", "d2", "parameter1", -1.1, "Base"], - ["Data", "d2", "parameter2", 2.3, "Base"], + ["Data", ("d1",), "parameter1", 1.1, "Base"], + ["Data", ("d1",), "parameter2", -2.3, "Base"], + ["Data", ("d2",), "parameter1", -1.1, "Base"], + ["Data", ("d2",), "parameter2", 2.3, "Base"], ], "parameter_definitions": [("Data", "parameter1"), ("Data", "parameter2")], "entities": [("Data", "d1"), ("Data", "d2")], @@ -498,12 +498,12 @@ def test_importing_multidimensional_class_when_there_is_an_extra_column(self): { "alternatives": {"Base"}, "entities": [ + ("unit__node__node", ["Dyson sphere", "Gamma Ceti", "Ring world"]), ("unit", "Dyson sphere"), ("node", "Gamma Ceti"), ("node", "Ring world"), - ("unit__node__node", ("Dyson sphere", "Gamma Ceti", "Ring world")), ], - "entity_classes": [("unit",), ("node",), ("unit__node__node", ("unit", "node", "node"))], + "entity_classes": [["unit__node__node", ["unit", "node", "node"]], ["unit", []], ["node", []]], "parameter_definitions": [("unit__node__node", "flow")], "parameter_values": [ ["unit__node__node", ("Dyson sphere", "Gamma Ceti", "Ring world"), "flow", 23.3, "Base"] @@ -535,9 +535,9 @@ def test_importing_empty_rows_does_unnecessarily_not_repeat_mapped_data(self): mapped_data, { "entities": [("Generator", "MyHydroGenerator")], - "entity_classes": [("Generator",)], + "entity_classes": [["Generator", []]], "parameter_definitions": [("Generator", "Type")], - "parameter_values": [["Generator", "MyHydroGenerator", "Type", "Hydro"]], + "parameter_values": [["Generator", ("MyHydroGenerator",), "Type", "Hydro"]], }, ) @@ -583,23 +583,23 @@ def test_pivoted_mapping_has_position_outside_source_bounds(self): mapped_data, { "entities": [ + ("connection__node__node", ["A1", "B1", "C1"]), ("connection", "A1"), ("node", "B1"), ("node", "C1"), - ("connection__node__node", ("A1", "B1", "C1")), + ("connection__node__node", ["A2", "B2", "C2"]), ("connection", "A2"), ("node", "B2"), ("node", "C2"), - ("connection__node__node", ("A2", "B2", "C2")), + ("connection__node__node", ["A3", "B3", "C3"]), ("connection", "A3"), ("node", "B3"), ("node", "C3"), - ("connection__node__node", ("A3", "B3", "C3")), ], "entity_classes": [ - ("connection",), - ("node",), - ("connection__node__node", ("connection", "node", "node")), + ["connection__node__node", ["connection", "node", "node"]], + ["connection", []], + ["node", []], ], "parameter_definitions": [("connection__node__node", "flow_t")], "parameter_values": [ @@ -677,15 +677,15 @@ def test_import_datetime_values(self): mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ ("Object", "o1"), ("Object", "o2"), ], "parameter_definitions": [("Object", "t")], "parameter_values": [ - ["Object", "o1", "t", DateTime("2024-06-24T09:00:00"), "Base"], - ["Object", "o2", "t", DateTime("2024-06-24T00:00:00"), "Base"], + ["Object", ("o1",), "t", DateTime("2024-06-24T09:00:00"), "Base"], + ["Object", ("o2",), "t", DateTime("2024-06-24T00:00:00"), "Base"], ], }, ) @@ -712,15 +712,15 @@ def test_import_durations(self): mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ ("Object", "o1"), ("Object", "o2"), ], "parameter_definitions": [("Object", "t")], "parameter_values": [ - ["Object", "o1", "t", Duration("23D"), "Base"], - ["Object", "o2", "t", Duration("19D"), "Base"], + ["Object", ("o1",), "t", Duration("23D"), "Base"], + ["Object", ("o2",), "t", Duration("19D"), "Base"], ], }, ) @@ -786,7 +786,7 @@ def test_import_entity_alternatives_with_activity_string(self): mapped_data, { "alternatives": {"Base", "alt1", "alt2"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ ("Object", "o1"), ], @@ -813,7 +813,7 @@ def test_import_entity_alternatives_with_activity_boolean(self): mapped_data, { "alternatives": {"Base", "alt1", "alt2"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ ("Object", "o1"), ], @@ -840,7 +840,7 @@ def test_import_entity_alternatives_with_activity_integer(self): mapped_data, { "alternatives": {"Base", "alt1", "alt2"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ ("Object", "o1"), ], @@ -873,7 +873,7 @@ def test_import_entity_alternatives_errors_gracefully_when_activity_cannot_be_co mapped_data, { "alternatives": {"Base"}, - "entity_classes": [("Object",)], + "entity_classes": [["Object", []]], "entities": [ ("Object", "o1"), ], @@ -904,13 +904,13 @@ def test_import_entity_alternatives_with_multidimensional_entities(self): mapped_data, { "alternatives": {"Base", "alt1"}, - "entity_classes": [("Widget",), ("Gadget",), ("Widget__Gadget", ("Widget", "Gadget"))], + "entity_classes": [["Widget__Gadget", ["Widget", "Gadget"]], ["Widget", []], ["Gadget", []]], "entities": [ + ("Widget__Gadget", ["o1", "p1"]), ("Widget", "o1"), ("Gadget", "p1"), - ("Widget__Gadget", ("o1", "p1")), + ("Widget__Gadget", ["o1", "p2"]), ("Gadget", "p2"), - ("Widget__Gadget", ("o1", "p2")), ], "entity_alternatives": [ ("Widget__Gadget", ("o1", "p1"), "Base", True), @@ -945,7 +945,7 @@ def test_import_parameter_types(self): self.assertEqual( mapped_data, { - "entity_classes": [("Widget",), ("Gadget",), ("Object",)], + "entity_classes": [["Widget", []], ["Gadget", []], ["Object", []]], "parameter_definitions": [("Widget", "x"), ("Gadget", "p"), ("Gadget", "q"), ("Object", "w")], "parameter_types": [ ("Widget", "x", "float"), @@ -1037,7 +1037,7 @@ def test_column_header_position_while_leaf_is_hidden(self): mapped_data, { "entity_classes": [ - ("Widget",), + ["Widget", []], ], "entities": [("Widget", "gadget")], }, @@ -1075,11 +1075,11 @@ def test_missing_entity_alternative_does_not_prevent_importing_of_values(self): "alternatives": {"Succeed", "Fail"}, "entities": [("unit", "Wind_plant")], "entity_alternatives": [("unit", ("Wind_plant",), "Succeed", True)], - "entity_classes": [("unit",)], + "entity_classes": [["unit", []]], "parameter_definitions": [("unit", "existing")], "parameter_values": [ - ["unit", "Wind_plant", "existing", 150.0, "Fail"], - ["unit", "Wind_plant", "existing", 200.0, "Succeed"], + ["unit", ("Wind_plant",), "existing", 150.0, "Fail"], + ["unit", ("Wind_plant",), "existing", 200.0, "Succeed"], ], }, ) @@ -1135,7 +1135,7 @@ def test_import_entity_metadata(self): mapped_data, { "entities": [("cat", "Garfield"), ("cat", "Tom")], - "entity_classes": [("cat",)], + "entity_classes": [["cat", []]], "entity_metadata": [ ("cat", ("Garfield",), "Created", "1976"), ("cat", ("Garfield",), "Keywords", "laziness, gluttony"), @@ -1171,7 +1171,7 @@ def test_import_parameter_value_metadata(self): { "alternatives": {"Base"}, "entities": [("cat", "Garfield"), ("cat", "Tom")], - "entity_classes": [("cat",)], + "entity_classes": [["cat", []]], "parameter_definitions": [("cat", "weight")], "parameter_value_metadata": [ ("cat", ("Garfield",), "weight", "Tools", "Harrison-Stetson 1.0", "Base"), @@ -1187,7 +1187,8 @@ def test_import_alternatives_with_descriptions(self): data_source = iter( [ ["alt1", "First alternative."], - ["alt2", None], + ["alt2", ""], + ["alt3", None], ["duplicate", "Overridden description."], ["duplicate", "Overriding description."], ] @@ -1203,6 +1204,7 @@ def test_import_alternatives_with_descriptions(self): "alternatives": { ("alt1", "First alternative."), "alt2", + "alt3", ("duplicate", "Overridden description."), ("duplicate", "Overriding description."), }, @@ -1215,6 +1217,7 @@ def test_import_scenarios_with_descriptions(self): [ ["scen1", "First scenario."], ["scen2", None], + ["scen3", ""], ["duplicate", "Possible description no. 1."], ["duplicate", "Possible description no. 2."], ] @@ -1230,8 +1233,47 @@ def test_import_scenarios_with_descriptions(self): "scenarios": { ("scen1", "First scenario."), ("scen2",), + ("scen3",), ("duplicate", "Possible description no. 1."), ("duplicate", "Possible description no. 2."), }, }, ) + + def test_import_entity_classes_with_description(self): + header = ["Class", "Description", "Entity"] + data_source = iter( + [ + ["unit", "Unit of production.", "coal_plant"], + ["node", "Nodes of processing.", "southern_hemisphere"], + ["node", "Nodes of processing.", "northern_hemisphere"], + ["model", None, "all_year_round"], + ["direction", "", "up"], + ["direction", "", "down"], + ] + ) + flattened = default_import_mapping("EntityClass").flatten() + flattened[0].position = 0 + flattened[1].position = 1 + flattened[2].position = 2 + mapped_data, errors = get_mapped_data(data_source, [to_dict(unflatten(flattened))], header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "entity_classes": [ + ["unit", [], "Unit of production."], + ["node", [], "Nodes of processing."], + ["model", []], + ["direction", []], + ], + "entities": [ + ("unit", "coal_plant"), + ("node", "southern_hemisphere"), + ("node", "northern_hemisphere"), + ("model", "all_year_round"), + ("direction", "up"), + ("direction", "down"), + ], + }, + ) diff --git a/tests/import_mapping/test_import_mapping.py b/tests/import_mapping/test_import_mapping.py index 2238760b..b8988398 100644 --- a/tests/import_mapping/test_import_mapping.py +++ b/tests/import_mapping/test_import_mapping.py @@ -60,7 +60,7 @@ def test_convert_functions_float(self): param_def_mapping.flatten()[-1].position = 1 mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { - "entity_classes": [("a",)], + "entity_classes": [["a", []]], "entities": [("a", "obj")], "parameter_definitions": [("a", "param", 1.2)], } @@ -79,7 +79,7 @@ def test_convert_functions_str(self): param_def_mapping.flatten()[-1].position = 1 mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { - "entity_classes": [("a",)], + "entity_classes": [["a", []]], "entities": [("a", "obj")], "parameter_definitions": [("a", "param", "1111.2222")], } @@ -98,7 +98,7 @@ def test_convert_functions_bool(self): param_def_mapping.flatten()[-1].position = 1 mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { - "entity_classes": [("a",)], + "entity_classes": [["a", []]], "entities": [("a", "obj")], "parameter_definitions": [("a", "param", False)], } @@ -782,7 +782,7 @@ def test_read_iterator_with_row_with_all_Nones(self): [None, None, None, None], ["oc2", "obj2", "parameter_name2", 2], ] - expected = {"entity_classes": [("oc2",)]} + expected = {"entity_classes": [["oc2", []]]} data = iter(input_data) data_header = next(data) @@ -795,7 +795,7 @@ def test_read_iterator_with_row_with_all_Nones(self): def test_read_iterator_with_None(self): input_data = [["object_class", "object", "parameter", "value"], None, ["oc2", "obj2", "parameter_name2", 2]] - expected = {"entity_classes": [("oc2",)]} + expected = {"entity_classes": [["oc2", []]]} data = iter(input_data) data_header = next(data) @@ -813,10 +813,10 @@ def test_read_flat_file(self): ["oc2", "obj2", "parameter_name2", 2], ] expected = { - "entity_classes": [("oc1",), ("oc2",)], + "entity_classes": [["oc1", []], ["oc2", []]], "entities": [("oc1", "obj1"), ("oc2", "obj2")], "parameter_definitions": [("oc1", "parameter_name1"), ("oc2", "parameter_name2")], - "parameter_values": [["oc1", "obj1", "parameter_name1", 1], ["oc2", "obj2", "parameter_name2", 2]], + "parameter_values": [["oc1", ("obj1",), "parameter_name1", 1], ["oc2", ("obj2",), "parameter_name2", 2]], } data = iter(input_data) @@ -840,10 +840,10 @@ def test_read_flat_file_array(self): ["oc1", "obj1", "parameter_name1", 2], ] expected = { - "entity_classes": [("oc1",)], + "entity_classes": [["oc1", []]], "entities": [("oc1", "obj1")], "parameter_definitions": [("oc1", "parameter_name1")], - "parameter_values": [["oc1", "obj1", "parameter_name1", Array([1, 2])]], + "parameter_values": [["oc1", ("obj1",), "parameter_name1", Array([1, 2])]], } data = iter(input_data) @@ -867,10 +867,10 @@ def test_read_flat_file_array_with_ed(self): ["oc1", "obj1", "parameter_name1", 2, 1], ] expected = { - "entity_classes": [("oc1",)], + "entity_classes": [["oc1", []]], "entities": [("oc1", "obj1")], "parameter_definitions": [("oc1", "parameter_name1")], - "parameter_values": [["oc1", "obj1", "parameter_name1", Array([1, 2])]], + "parameter_values": [["oc1", ("obj1",), "parameter_name1", Array([1, 2])]], } data = iter(input_data) @@ -895,7 +895,7 @@ def test_read_flat_file_array_with_ed(self): def test_read_flat_file_with_column_name_reference(self): input_data = [["object", "parameter", "value"], ["obj1", "parameter_name1", 1], ["obj2", "parameter_name2", 2]] - expected = {"entity_classes": [("object",)], "entities": [("object", "obj1"), ("object", "obj2")]} + expected = {"entity_classes": [["object", []]], "entities": [("object", "obj1"), ("object", "obj2")]} data = iter(input_data) data_header = next(data) @@ -909,7 +909,7 @@ def test_read_flat_file_with_column_name_reference(self): def test_read_object_class_from_header_using_string_as_integral_index(self): input_data = [["object_class"], ["obj1"], ["obj2"]] expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "entities": [("object_class", "obj1"), ("object_class", "obj2")], } @@ -925,7 +925,7 @@ def test_read_object_class_from_header_using_string_as_integral_index(self): def test_read_object_class_from_header_using_string_as_column_header_name(self): input_data = [["object_class"], ["obj1"], ["obj2"]] expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "entities": [("object_class", "obj1"), ("object_class", "obj2")], } @@ -944,7 +944,7 @@ def test_read_object_class_from_header_using_string_as_column_header_name(self): def test_read_with_list_of_mappings(self): input_data = [["object", "parameter", "value"], ["obj1", "parameter_name1", 1], ["obj2", "parameter_name2", 2]] - expected = {"entity_classes": [("object",)], "entities": [("object", "obj1"), ("object", "obj2")]} + expected = {"entity_classes": [["object", []]], "entities": [("object", "obj1"), ("object", "obj2")]} data = iter(input_data) data_header = next(data) @@ -958,14 +958,14 @@ def test_read_with_list_of_mappings(self): def test_read_pivoted_parameters_from_header(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { - "entity_classes": [("object",)], + "entity_classes": [["object", []]], "entities": [("object", "obj1"), ("object", "obj2")], "parameter_definitions": [("object", "parameter_name1"), ("object", "parameter_name2")], "parameter_values": [ - ["object", "obj1", "parameter_name1", 0], - ["object", "obj1", "parameter_name2", 1], - ["object", "obj2", "parameter_name1", 2], - ["object", "obj2", "parameter_name2", 3], + ["object", ("obj1",), "parameter_name1", 0], + ["object", ("obj1",), "parameter_name2", 1], + ["object", ("obj2",), "parameter_name1", 2], + ["object", ("obj2",), "parameter_name2", 3], ], } @@ -1004,14 +1004,14 @@ def test_read_empty_pivot(self): def test_read_pivoted_parameters_from_data(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { - "entity_classes": [("object",)], + "entity_classes": [["object", []]], "entities": [("object", "obj1"), ("object", "obj2")], "parameter_definitions": [("object", "parameter_name1"), ("object", "parameter_name2")], "parameter_values": [ - ["object", "obj1", "parameter_name1", 0], - ["object", "obj1", "parameter_name2", 1], - ["object", "obj2", "parameter_name1", 2], - ["object", "obj2", "parameter_name2", 3], + ["object", ("obj1",), "parameter_name1", 0], + ["object", ("obj1",), "parameter_name2", 1], + ["object", ("obj2",), "parameter_name1", 2], + ["object", ("obj2",), "parameter_name2", 3], ], } @@ -1038,13 +1038,13 @@ def test_pivoted_value_has_actual_position(self): ["obj2", "T2", 22.0], ] expected = { - "entity_classes": [("timeline",)], + "entity_classes": [["timeline", []]], "entities": [("timeline", "obj1"), ("timeline", "obj2")], "parameter_definitions": [("timeline", "value")], "alternatives": {"Base"}, "parameter_values": [ - ["timeline", "obj1", "value", Map(["T1", "T2"], [11.0, 12.0], index_name="timestep"), "Base"], - ["timeline", "obj2", "value", Map(["T1", "T2"], [21.0, 22.0], index_name="timestep"), "Base"], + ["timeline", ("obj1",), "value", Map(["T1", "T2"], [11.0, 12.0], index_name="timestep"), "Base"], + ["timeline", ("obj2",), "value", Map(["T1", "T2"], [21.0, 22.0], index_name="timestep"), "Base"], ], } data = iter(input_data) @@ -1068,13 +1068,13 @@ def test_import_objects_from_pivoted_data_when_they_lack_parameter_values(self): """Pivoted mapping works even when last mapping has valid position in columns.""" input_data = [["object", "is_skilled", "has_powers"], ["obj1", "yes", "no"], ["obj2", None, None]] expected = { - "entity_classes": [("node",)], + "entity_classes": [["node", []]], "entities": [("node", "obj1"), ("node", "obj2")], "parameter_definitions": [("node", "is_skilled"), ("node", "has_powers")], "alternatives": {"Base"}, "parameter_values": [ - ["node", "obj1", "is_skilled", "yes", "Base"], - ["node", "obj1", "has_powers", "no", "Base"], + ["node", ("obj1",), "is_skilled", "yes", "Base"], + ["node", ("obj1",), "has_powers", "no", "Base"], ], } data = iter(input_data) @@ -1099,12 +1099,18 @@ def test_import_objects_from_pivoted_data_when_they_lack_map_type_parameter_valu ["obj1", "today", None, "yes"], ] expected = { - "entity_classes": [("node",)], + "entity_classes": [["node", []]], "entities": [("node", "obj1")], "parameter_definitions": [("node", "is_skilled"), ("node", "has_powers")], "alternatives": {"Base"}, "parameter_values": [ - ["node", "obj1", "has_powers", Map(["yesterday", "today"], ["no", "yes"], index_name="period"), "Base"] + [ + "node", + ("obj1",), + "has_powers", + Map(["yesterday", "today"], ["no", "yes"], index_name="period"), + "Base", + ] ], } data = iter(input_data) @@ -1128,13 +1134,13 @@ def test_read_flat_file_with_extra_value_dimensions(self): input_data = [["object", "time", "parameter_name1"], ["obj1", "2018-01-01", 1], ["obj1", "2018-01-02", 2]] expected = { - "entity_classes": [("object",)], + "entity_classes": [["object", []]], "entities": [("object", "obj1")], "parameter_definitions": [("object", "parameter_name1")], "parameter_values": [ [ "object", - "obj1", + ("obj1",), "parameter_name1", TimeSeriesVariableResolution(["2018-01-01", "2018-01-02"], [1, 2], False, False), ] @@ -1165,7 +1171,7 @@ def test_read_flat_file_with_parameter_definition(self): input_data = [["object", "time", "parameter_name1"], ["obj1", "2018-01-01", 1], ["obj1", "2018-01-02", 2]] expected = { - "entity_classes": [("object",)], + "entity_classes": [["object", []]], "entities": [("object", "obj1")], "parameter_definitions": [("object", "parameter_name1")], } @@ -1192,8 +1198,8 @@ def test_read_flat_file_with_parameter_definition(self): def test_read_1dim_relationships(self): input_data = [["unit", "node"], ["u1", "n1"], ["u1", "n2"]] expected = { - "entity_classes": [("node_group", ("node",))], - "entities": [("node_group", ("n1",)), ("node_group", ("n2",))], + "entity_classes": [["node_group", ["node"]]], + "entities": [("node_group", ["n1"]), ("node_group", ["n2"])], } data = iter(input_data) @@ -1213,8 +1219,8 @@ def test_read_1dim_relationships(self): def test_read_relationships(self): input_data = [["unit", "node"], ["u1", "n1"], ["u1", "n2"]] expected = { - "entity_classes": [("unit__node", ("unit", "node"))], - "entities": [("unit__node", ("u1", "n1")), ("unit__node", ("u1", "n2"))], + "entity_classes": [["unit__node", ["unit", "node"]]], + "entities": [("unit__node", ["u1", "n1"]), ("unit__node", ["u1", "n2"])], } data = iter(input_data) @@ -1237,8 +1243,8 @@ def test_read_relationships(self): def test_read_relationships_with_parameters(self): input_data = [["unit", "node", "rel_parameter"], ["u1", "n1", 0], ["u1", "n2", 1]] expected = { - "entity_classes": [("unit__node", ("unit", "node"))], - "entities": [("unit__node", ("u1", "n1")), ("unit__node", ("u1", "n2"))], + "entity_classes": [["unit__node", ["unit", "node"]]], + "entities": [("unit__node", ["u1", "n1"]), ("unit__node", ["u1", "n2"])], "parameter_definitions": [("unit__node", "rel_parameter")], "parameter_values": [ ["unit__node", ("u1", "n1"), "rel_parameter", 0], @@ -1267,13 +1273,13 @@ def test_read_relationships_with_parameters(self): def test_read_relationships_with_parameters2(self): input_data = [["nuts2", "Capacity", "Fueltype"], ["BE23", 268.0, "Bioenergy"], ["DE11", 14.0, "Bioenergy"]] expected = { - "entity_classes": [("nuts2",), ("fueltype",), ("nuts2__fueltype", ("nuts2", "fueltype"))], + "entity_classes": [["nuts2__fueltype", ["nuts2", "fueltype"]], ["nuts2", []], ["fueltype", []]], "entities": [ + ("nuts2__fueltype", ["BE23", "Bioenergy"]), ("nuts2", "BE23"), ("fueltype", "Bioenergy"), - ("nuts2__fueltype", ("BE23", "Bioenergy")), + ("nuts2__fueltype", ["DE11", "Bioenergy"]), ("nuts2", "DE11"), - ("nuts2__fueltype", ("DE11", "Bioenergy")), ], "parameter_definitions": [("nuts2__fueltype", "capacity")], "parameter_values": [ @@ -1311,12 +1317,12 @@ def test_read_relationships_with_parameters2(self): def test_read_parameter_header_with_only_one_parameter(self): input_data = [["object", "parameter_name1"], ["obj1", 0], ["obj2", 2]] expected = { - "entity_classes": [("object",)], + "entity_classes": [["object", []]], "entities": [("object", "obj1"), ("object", "obj2")], "parameter_definitions": [("object", "parameter_name1")], "parameter_values": [ - ["object", "obj1", "parameter_name1", 0], - ["object", "obj2", "parameter_name1", 2], + ["object", ("obj1",), "parameter_name1", 0], + ["object", ("obj2",), "parameter_name1", 2], ], } @@ -1337,12 +1343,12 @@ def test_read_parameter_header_with_only_one_parameter(self): def test_read_pivoted_parameters_from_data_with_skipped_column(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { - "entity_classes": [("object",)], + "entity_classes": [["object", []]], "entities": [("object", "obj1"), ("object", "obj2")], "parameter_definitions": [("object", "parameter_name1")], "parameter_values": [ - ["object", "obj1", "parameter_name1", 0], - ["object", "obj2", "parameter_name1", 2], + ["object", ("obj1",), "parameter_name1", 0], + ["object", ("obj2",), "parameter_name1", 2], ], } @@ -1363,14 +1369,18 @@ def test_read_pivoted_parameters_from_data_with_skipped_column(self): def test_read_relationships_and_import_objects(self): input_data = [["unit", "node"], ["u1", "n1"], ["u2", "n2"]] expected = { - "entity_classes": [("unit",), ("node",), ("unit__node", ("unit", "node"))], + "entity_classes": [ + ["unit__node", ["unit", "node"]], + ["unit", []], + ["node", []], + ], "entities": [ + ("unit__node", ["u1", "n1"]), ("unit", "u1"), ("node", "n1"), - ("unit__node", ("u1", "n1")), + ("unit__node", ["u2", "n2"]), ("unit", "u2"), ("node", "n2"), - ("unit__node", ("u2", "n2")), ], } @@ -1394,11 +1404,10 @@ def test_read_relationships_and_import_objects(self): def test_read_relationships_parameter_values_with_extra_dimensions(self): input_data = [["", "a", "b"], ["", "c", "d"], ["", "e", "f"], ["a", 2, 3], ["b", 4, 5]] - expected = { - "entity_classes": [("unit__node", ("unit", "node"))], + "entity_classes": [["unit__node", ["unit", "node"]]], "parameter_definitions": [("unit__node", "e"), ("unit__node", "f")], - "entities": [("unit__node", ("a", "c")), ("unit__node", ("b", "d"))], + "entities": [("unit__node", ["a", "c"]), ("unit__node", ["b", "d"])], "parameter_values": [ ["unit__node", ("a", "c"), "e", Map(["a", "b"], [2, 4])], ["unit__node", ("b", "d"), "f", Map(["a", "b"], [3, 5])], @@ -1433,10 +1442,10 @@ def test_read_data_with_read_start_row(self): ["oc2", "obj2", "parameter_name2", 2], ] expected = { - "entity_classes": [("oc1",), ("oc2",)], + "entity_classes": [["oc1", []], ["oc2", []]], "entities": [("oc1", "obj1"), ("oc2", "obj2")], "parameter_definitions": [("oc1", "parameter_name1"), ("oc2", "parameter_name2")], - "parameter_values": [["oc1", "obj1", "parameter_name1", 1], ["oc2", "obj2", "parameter_name2", 2]], + "parameter_values": [["oc1", ("obj1",), "parameter_name1", 1], ["oc2", ("obj2",), "parameter_name2", 2]], } data = iter(input_data) @@ -1462,13 +1471,13 @@ def test_read_data_with_two_mappings_with_different_read_start_row(self): ["oc1_obj2", "oc2_obj2", 2, 4], ] expected = { - "entity_classes": [("oc1",), ("oc2",)], + "entity_classes": [["oc1", []], ["oc2", []]], "entities": [("oc1", "oc1_obj1"), ("oc1", "oc1_obj2"), ("oc2", "oc2_obj2")], "parameter_definitions": [("oc1", "parameter_class1"), ("oc2", "parameter_class2")], "parameter_values": [ - ["oc1", "oc1_obj1", "parameter_class1", 1], - ["oc1", "oc1_obj2", "parameter_class1", 2], - ["oc2", "oc2_obj2", "parameter_class2", 4], + ["oc1", ("oc1_obj1",), "parameter_class1", 1], + ["oc1", ("oc1_obj2",), "parameter_class1", 2], + ["oc2", ("oc2_obj2",), "parameter_class2", 4], ], } @@ -1505,7 +1514,7 @@ def test_read_object_class_with_table_name_as_class_name(self): } out, errors = get_mapped_data(data, [mapping], data_header, "class name") expected = { - "entity_classes": [("class name",)], + "entity_classes": [["class name", []]], "entities": [("class name", "object 1"), ("class name", "object 2")], } self.assertFalse(errors) @@ -1530,9 +1539,9 @@ def test_read_flat_map_from_columns(self): out, errors = get_mapped_data(data, [mapping], data_header) expected_map = Map(["key1", "key2"], [-2, -1]) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } self.assertFalse(errors) @@ -1557,9 +1566,9 @@ def test_read_nested_map_from_columns(self): out, errors = get_mapped_data(data, [mapping], data_header) expected_map = Map(["key11", "key21"], [Map(["key12"], [-2]), Map(["key22"], [-1])]) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } self.assertFalse(errors) @@ -1601,9 +1610,9 @@ def test_read_uneven_nested_map_from_columns(self): ], ) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } self.assertFalse(errors) @@ -1643,9 +1652,9 @@ def test_read_nested_map_with_compression(self): ], ) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } self.assertFalse(errors) @@ -1795,12 +1804,13 @@ def test_read_object_group_without_parameters(self): data_header = next(data) mapping = {"map_type": "ObjectGroup", "name": 0, "groups": 1, "members": 2} out, errors = get_mapped_data(data, [mapping], data_header) - expected = {} - expected["entity_classes"] = [("class_A",)] - expected["entity_groups"] = { - ("class_A", "group1", "object1"), - ("class_A", "group1", "object2"), - ("class_A", "group2", "object3"), + expected = { + "entity_classes": [["class_A", []]], + "entity_groups": { + ("class_A", "group1", "object1"), + ("class_A", "group1", "object2"), + ("class_A", "group2", "object3"), + }, } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1816,20 +1826,21 @@ def test_read_object_group_and_import_objects(self): data_header = next(data) mapping = {"map_type": "ObjectGroup", "name": 0, "groups": 1, "members": 2, "import_objects": True} out, errors = get_mapped_data(data, [mapping], data_header) - expected = {} - expected["entity_groups"] = { - ("class_A", "group1", "object1"), - ("class_A", "group1", "object2"), - ("class_A", "group2", "object3"), - } - expected["entity_classes"] = [("class_A",)] - expected["entities"] = [ - ("class_A", "group1"), - ("class_A", "object1"), - ("class_A", "object2"), - ("class_A", "group2"), - ("class_A", "object3"), - ] + expected = { + "entity_groups": { + ("class_A", "group1", "object1"), + ("class_A", "group1", "object2"), + ("class_A", "group2", "object3"), + }, + "entity_classes": [["class_A", []]], + "entities": [ + ("class_A", "group1"), + ("class_A", "object1"), + ("class_A", "object2"), + ("class_A", "group2"), + ("class_A", "object3"), + ], + } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1853,13 +1864,14 @@ def test_read_parameter_definition_with_default_values_and_value_lists(self): }, } out, errors = get_mapped_data(data, [mapping], data_header) - expected = {} - expected["entity_classes"] = [("class_A",), ("class_B",)] - expected["parameter_definitions"] = [ - ("class_A", "param1", 23.0, "listA"), - ("class_A", "param2", 42.0, "listB"), - ("class_B", "param3", 5.0, "listA"), - ] + expected = { + "entity_classes": [["class_A", []], ["class_B", []]], + "parameter_definitions": [ + ("class_A", "param1", 23.0, "listA"), + ("class_A", "param2", 42.0, "listB"), + ("class_B", "param3", 5.0, "listA"), + ], + } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1878,7 +1890,7 @@ def test_map_as_default_parameter_value(self): out, errors = get_mapped_data(data, [mapping]) expected_map = Map(["key1", "key2", "key3"], [-2.3, 5.5, 3.2]) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "parameter_definitions": [("object_class", "parameter", expected_map)], } self.assertFalse(errors) @@ -1900,7 +1912,7 @@ def test_read_parameter_definition_with_nested_map_as_default_value(self): out, errors = get_mapped_data(data, [mapping], data_header) expected_map = Map(["key11", "key21"], [Map(["key12"], [-2]), Map(["key22"], [-1])]) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "parameter_definitions": [("object_class", "parameter", expected_map)], } self.assertFalse(errors) @@ -1930,9 +1942,9 @@ def test_read_map_index_names_from_columns(self): index_name="Index 1", ) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } self.assertFalse(errors) @@ -1962,9 +1974,9 @@ def test_missing_map_index_name(self): index_name="", ) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "entities": [("object_class", "object")], - "parameter_values": [["object_class", "object", "parameter", expected_map]], + "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } self.assertFalse(errors) @@ -1993,7 +2005,7 @@ def test_read_default_value_index_names_from_columns(self): index_name="Index 1", ) expected = { - "entity_classes": [("object_class",)], + "entity_classes": [["object_class", []]], "parameter_definitions": [("object_class", "parameter", expected_map)], } self.assertFalse(errors) @@ -2004,7 +2016,7 @@ def test_filter_regular_expression_in_root_mapping(self): data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0, filter_re="B"), EntityMapping(1)]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [("B",)], "entities": [("B", "r")]} + expected = {"entity_classes": [["B", []]], "entities": [("B", "r")]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2013,7 +2025,7 @@ def test_filter_regular_expression_in_child_mapping(self): data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0), EntityMapping(1, filter_re="q|r")]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [("A",), ("B",)], "entities": [("A", "q"), ("B", "r")]} + expected = {"entity_classes": [["A", []], ["B", []]], "entities": [("A", "q"), ("B", "r")]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2022,7 +2034,7 @@ def test_filter_regular_expression_in_child_mapping_filters_parent_mappings_too( data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0), EntityMapping(1, filter_re="q")]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [("A",)], "entities": [("A", "q")]} + expected = {"entity_classes": [["A", []]], "entities": [("A", "q")]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2041,13 +2053,13 @@ def test_arrays_get_imported_to_correct_alternatives(self): ) out, errors = get_mapped_data(data, [mapping_root]) expected = { - "entity_classes": [("class",)], + "entity_classes": [["class", []]], "entities": [("class", "y")], "parameter_definitions": [("class", "parameter")], "alternatives": {"Base", "alternative"}, "parameter_values": [ - ["class", "y", "parameter", Array(["p1"]), "Base"], - ["class", "y", "parameter", Array(["p1"]), "alternative"], + ["class", ("y",), "parameter", Array(["p1"]), "Base"], + ["class", ("y",), "parameter", Array(["p1"]), "alternative"], ], } self.assertFalse(errors) diff --git a/tests/spine_io/importers/test_reader.py b/tests/spine_io/importers/test_reader.py index 99f03ce1..5d158eed 100644 --- a/tests/spine_io/importers/test_reader.py +++ b/tests/spine_io/importers/test_reader.py @@ -58,7 +58,7 @@ def test_get_mapped_data(self): table_row_convert_specs, ) self.assertEqual(errors, []) - self.assertEqual(mapped_data, {"entity_classes": [("A",)], "entities": [("A", "b")]}) + self.assertEqual(mapped_data, {"entity_classes": [["A", []]], "entities": [("A", "b")]}) def test_resolve_values_for_fixed_position_mappings(self): reader = Reader(None) @@ -126,7 +126,3 @@ def raise_exception(*args): ) self.assertEqual(errors, ["this is expected"]) self.assertEqual(mapped_data, {}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/spine_io/test_excel_integration.py b/tests/spine_io/test_excel_integration.py index 713a1e0e..63d4002a 100644 --- a/tests/spine_io/test_excel_integration.py +++ b/tests/spine_io/test_excel_integration.py @@ -10,7 +10,7 @@ # this program. If not, see . ###################################################################################################################### -""" Integration tests for Excel import and export. """ +"""Integration tests for Excel import and export.""" import json from pathlib import PurePath @@ -101,10 +101,10 @@ def test_map(self): def _check_parameter_value(self, val): input_data = { - "entity_classes": {("dog",)}, + "entity_classes": {("dog", ())}, "entities": {("dog", "pluto")}, "parameter_definitions": [("dog", "bone")], - "parameter_values": [("dog", "pluto", "bone", val)], + "parameter_values": [("dog", ("pluto",), "bone", val)], } with DatabaseMapping("sqlite://", create=True) as db_map: self._assert_imports(import_data(db_map, **input_data)) @@ -133,7 +133,3 @@ def indexed_values(value, k=1, prefix=()): yield from indexed_values(new_value, k=k + 1, prefix=(*prefix, str(index))) except AttributeError: yield str(prefix), value - - -if __name__ == "__main__": - unittest.main() From 9aba6dee8c9ee114a0c40ed3f666d066789b1e9e Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Wed, 25 Feb 2026 11:03:17 +0200 Subject: [PATCH 4/9] Add import mappings for entity descriptions Re spine-tools/Spine-Toolbox#1892 --- spinedb_api/import_mapping/generator.py | 5 +- spinedb_api/import_mapping/import_mapping.py | 22 ++- tests/import_mapping/test_generator.py | 181 +++++++++++++------ tests/import_mapping/test_import_mapping.py | 104 +++++------ tests/spine_io/importers/test_reader.py | 2 +- 5 files changed, 202 insertions(+), 112 deletions(-) diff --git a/spinedb_api/import_mapping/generator.py b/spinedb_api/import_mapping/generator.py index 3dde0ad5..29922b49 100644 --- a/spinedb_api/import_mapping/generator.py +++ b/spinedb_api/import_mapping/generator.py @@ -316,7 +316,10 @@ def _make_entities(mapped_data): return final_rows = [] for (class_name, name), record in rows.items(): - final_rows.append((class_name, name if not record.elements else record.elements)) + item = [class_name, name if not record.elements else record.elements] + if record.description: + item.append(record.description) + final_rows.append(item) if final_rows: mapped_data["entities"] = final_rows diff --git a/spinedb_api/import_mapping/import_mapping.py b/spinedb_api/import_mapping/import_mapping.py index 2549a2a8..19f5fd73 100644 --- a/spinedb_api/import_mapping/import_mapping.py +++ b/spinedb_api/import_mapping/import_mapping.py @@ -436,6 +436,7 @@ def _import_row(self, source_data, state, mapped_data): @dataclass class EntityRecord: elements: list[str] = field(default_factory=list) + description: str | None = None class EntityMapping(ImportMapping): @@ -454,6 +455,23 @@ def _import_row(self, source_data, state, mapped_data): mapped_data.setdefault("entities", {})[entity_class_name, entity_name] = EntityRecord() +class EntityDescriptionMapping(ImportMapping): + """Maps entity descriptions. + + Cannot be used as the topmost mapping; one of the parents must be :class:`EntityMapping` or :class:`ElementMapping`. + """ + + MAP_TYPE = "EntityDescription" + ignorable = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + entity_name = state[ImportKey.ENTITY_NAME] + mapped_data["entities"][entity_class_name, entity_name].description = description + + class EntityMetadataNameMapping(ImportMapping): """Maps entity metadata names.""" @@ -1050,7 +1068,8 @@ def _default_entity_class_mapping() -> EntityClassMapping: """ root_mapping = EntityClassMapping(Position.hidden) description_mapping = root_mapping.child = EntityClassDescriptionMapping(Position.hidden) - description_mapping.child = EntityMapping(Position.hidden) + entity_mapping = description_mapping.child = EntityMapping(Position.hidden) + entity_mapping.child = EntityDescriptionMapping(Position.hidden) return root_mapping @@ -1154,6 +1173,7 @@ def from_dict(serialized): EntityClassMapping, EntityClassDescriptionMapping, EntityMapping, + EntityDescriptionMapping, EntityMetadataNameMapping, EntityMetadataValueMapping, EntityGroupMapping, diff --git a/tests/import_mapping/test_generator.py b/tests/import_mapping/test_generator.py index e92c9a9f..a0f58268 100644 --- a/tests/import_mapping/test_generator.py +++ b/tests/import_mapping/test_generator.py @@ -72,7 +72,7 @@ def test_returns_appropriate_error_if_last_row_is_empty(self): "entity_classes": [["Object", []]], "parameter_values": [["Object", ("data",), "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], "parameter_definitions": [("Object", "Parameter")], - "entities": [("Object", "data")], + "entities": [["Object", "data"]], }, ) @@ -104,7 +104,7 @@ def test_convert_functions_get_expanded_over_last_defined_column_in_pivoted_data "entity_classes": [["Object", []]], "parameter_values": [["Object", ("data",), "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], "parameter_definitions": [("Object", "Parameter")], - "entities": [("Object", "data")], + "entities": [["Object", "data"]], }, ) @@ -135,7 +135,7 @@ def test_read_start_row_skips_rows_in_pivoted_data(self): "entity_classes": [["klass", []]], "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], "parameter_definitions": [("klass", "Parameter_2")], - "entities": [("klass", "kloss")], + "entities": [["klass", "kloss"]], }, ) @@ -189,7 +189,7 @@ def test_map_without_values_is_ignored_and_not_interpreted_as_null(self): "entity_classes": [["o", []]], "parameter_definitions": [("o", "parameter_name")], "parameter_values": [], - "entities": [("o", "o1")], + "entities": [["o", "o1"]], }, ) @@ -224,13 +224,13 @@ def test_import_object_works_with_multiple_relationship_object_imports(self): "alternatives": {"base"}, "entity_classes": [["o_to_q", ["o", "q"]], ["o", []], ["q", []]], "entities": [ - ("o_to_q", ["o1", "q1"]), - ("o", "o1"), - ("q", "q1"), - ("o_to_q", ["o2", "q2"]), - ("o", "o2"), - ("q", "q2"), - ("o_to_q", ["o1", "q2"]), + ["o_to_q", ["o1", "q1"]], + ["o", "o1"], + ["q", "q1"], + ["o_to_q", ["o2", "q2"]], + ["o", "o2"], + ["q", "q2"], + ["o_to_q", ["o1", "q2"]], ], "parameter_definitions": [("o_to_q", "param")], "parameter_values": [ @@ -269,7 +269,7 @@ def test_default_convert_function_in_column_convert_functions(self): "entity_classes": [["klass", []]], "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], "parameter_definitions": [("klass", "Parameter_2")], - "entities": [("klass", "kloss")], + "entities": [["klass", "kloss"]], }, ) @@ -297,7 +297,7 @@ def test_identity_function_is_used_as_convert_function_when_no_convert_functions "entity_classes": [["klass", []]], "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], ["2.3", "23.0"])]], "parameter_definitions": [("klass", "Parameter_2")], - "entities": [("klass", "kloss")], + "entities": [["klass", "kloss"]], }, ) @@ -327,7 +327,7 @@ def test_last_convert_function_gets_used_as_default_convert_function_when_no_def "entity_classes": [["klass", []]], "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], "parameter_definitions": [("klass", "Parameter_2")], - "entities": [("klass", "kloss")], + "entities": [["klass", "kloss"]], }, ) @@ -363,7 +363,7 @@ def test_array_parameters_get_imported_correctly_when_objects_are_in_header(self ["class", ("object_2",), "param", Array([2.3, -2.3]), "Base"], ], "parameter_definitions": [("class", "param")], - "entities": [("class", "object_1"), ("class", "object_2")], + "entities": [["class", "object_1"], ["class", "object_2"]], }, ) @@ -399,7 +399,7 @@ def test_arrays_get_imported_correctly_when_objects_are_in_header_and_alternativ ["Gadget", ("object_2",), "data", Array([2.3, -2.3]), "Base"], ], "parameter_definitions": [("Gadget", "data")], - "entities": [("Gadget", "object_1"), ("Gadget", "object_2")], + "entities": [["Gadget", "object_1"], ["Gadget", "object_2"]], }, ) @@ -436,7 +436,7 @@ def test_header_position_is_ignored_in_last_mapping_if_other_mappings_are_in_hea ["Data", ("d2",), "parameter2", 2.3, "Base"], ], "parameter_definitions": [("Data", "parameter1"), ("Data", "parameter2")], - "entities": [("Data", "d1"), ("Data", "d2")], + "entities": [["Data", "d1"], ["Data", "d2"]], }, ) @@ -498,10 +498,10 @@ def test_importing_multidimensional_class_when_there_is_an_extra_column(self): { "alternatives": {"Base"}, "entities": [ - ("unit__node__node", ["Dyson sphere", "Gamma Ceti", "Ring world"]), - ("unit", "Dyson sphere"), - ("node", "Gamma Ceti"), - ("node", "Ring world"), + ["unit__node__node", ["Dyson sphere", "Gamma Ceti", "Ring world"]], + ["unit", "Dyson sphere"], + ["node", "Gamma Ceti"], + ["node", "Ring world"], ], "entity_classes": [["unit__node__node", ["unit", "node", "node"]], ["unit", []], ["node", []]], "parameter_definitions": [("unit__node__node", "flow")], @@ -534,7 +534,7 @@ def test_importing_empty_rows_does_unnecessarily_not_repeat_mapped_data(self): self.assertEqual( mapped_data, { - "entities": [("Generator", "MyHydroGenerator")], + "entities": [["Generator", "MyHydroGenerator"]], "entity_classes": [["Generator", []]], "parameter_definitions": [("Generator", "Type")], "parameter_values": [["Generator", ("MyHydroGenerator",), "Type", "Hydro"]], @@ -583,18 +583,18 @@ def test_pivoted_mapping_has_position_outside_source_bounds(self): mapped_data, { "entities": [ - ("connection__node__node", ["A1", "B1", "C1"]), - ("connection", "A1"), - ("node", "B1"), - ("node", "C1"), - ("connection__node__node", ["A2", "B2", "C2"]), - ("connection", "A2"), - ("node", "B2"), - ("node", "C2"), - ("connection__node__node", ["A3", "B3", "C3"]), - ("connection", "A3"), - ("node", "B3"), - ("node", "C3"), + ["connection__node__node", ["A1", "B1", "C1"]], + ["connection", "A1"], + ["node", "B1"], + ["node", "C1"], + ["connection__node__node", ["A2", "B2", "C2"]], + ["connection", "A2"], + ["node", "B2"], + ["node", "C2"], + ["connection__node__node", ["A3", "B3", "C3"]], + ["connection", "A3"], + ["node", "B3"], + ["node", "C3"], ], "entity_classes": [ ["connection__node__node", ["connection", "node", "node"]], @@ -679,8 +679,8 @@ def test_import_datetime_values(self): "alternatives": {"Base"}, "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), - ("Object", "o2"), + ["Object", "o1"], + ["Object", "o2"], ], "parameter_definitions": [("Object", "t")], "parameter_values": [ @@ -714,8 +714,8 @@ def test_import_durations(self): "alternatives": {"Base"}, "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), - ("Object", "o2"), + ["Object", "o1"], + ["Object", "o2"], ], "parameter_definitions": [("Object", "t")], "parameter_values": [ @@ -788,7 +788,7 @@ def test_import_entity_alternatives_with_activity_string(self): "alternatives": {"Base", "alt1", "alt2"}, "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), + ["Object", "o1"], ], "entity_alternatives": [("Object", ("o1",), "Base", True), ("Object", ("o1",), "alt1", False)], }, @@ -815,7 +815,7 @@ def test_import_entity_alternatives_with_activity_boolean(self): "alternatives": {"Base", "alt1", "alt2"}, "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), + ["Object", "o1"], ], "entity_alternatives": [("Object", ("o1",), "Base", True), ("Object", ("o1",), "alt1", False)], }, @@ -842,7 +842,7 @@ def test_import_entity_alternatives_with_activity_integer(self): "alternatives": {"Base", "alt1", "alt2"}, "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), + ["Object", "o1"], ], "entity_alternatives": [("Object", ("o1",), "Base", True), ("Object", ("o1",), "alt1", False)], }, @@ -875,7 +875,7 @@ def test_import_entity_alternatives_errors_gracefully_when_activity_cannot_be_co "alternatives": {"Base"}, "entity_classes": [["Object", []]], "entities": [ - ("Object", "o1"), + ["Object", "o1"], ], "entity_alternatives": [], }, @@ -906,11 +906,11 @@ def test_import_entity_alternatives_with_multidimensional_entities(self): "alternatives": {"Base", "alt1"}, "entity_classes": [["Widget__Gadget", ["Widget", "Gadget"]], ["Widget", []], ["Gadget", []]], "entities": [ - ("Widget__Gadget", ["o1", "p1"]), - ("Widget", "o1"), - ("Gadget", "p1"), - ("Widget__Gadget", ["o1", "p2"]), - ("Gadget", "p2"), + ["Widget__Gadget", ["o1", "p1"]], + ["Widget", "o1"], + ["Gadget", "p1"], + ["Widget__Gadget", ["o1", "p2"]], + ["Gadget", "p2"], ], "entity_alternatives": [ ("Widget__Gadget", ("o1", "p1"), "Base", True), @@ -1039,7 +1039,7 @@ def test_column_header_position_while_leaf_is_hidden(self): "entity_classes": [ ["Widget", []], ], - "entities": [("Widget", "gadget")], + "entities": [["Widget", "gadget"]], }, ) @@ -1073,7 +1073,7 @@ def test_missing_entity_alternative_does_not_prevent_importing_of_values(self): mapped_data, { "alternatives": {"Succeed", "Fail"}, - "entities": [("unit", "Wind_plant")], + "entities": [["unit", "Wind_plant"]], "entity_alternatives": [("unit", ("Wind_plant",), "Succeed", True)], "entity_classes": [["unit", []]], "parameter_definitions": [("unit", "existing")], @@ -1134,7 +1134,7 @@ def test_import_entity_metadata(self): self.assertEqual( mapped_data, { - "entities": [("cat", "Garfield"), ("cat", "Tom")], + "entities": [["cat", "Garfield"], ["cat", "Tom"]], "entity_classes": [["cat", []]], "entity_metadata": [ ("cat", ("Garfield",), "Created", "1976"), @@ -1170,7 +1170,7 @@ def test_import_parameter_value_metadata(self): mapped_data, { "alternatives": {"Base"}, - "entities": [("cat", "Garfield"), ("cat", "Tom")], + "entities": [["cat", "Garfield"], ["cat", "Tom"]], "entity_classes": [["cat", []]], "parameter_definitions": [("cat", "weight")], "parameter_value_metadata": [ @@ -1268,12 +1268,79 @@ def test_import_entity_classes_with_description(self): ["direction", []], ], "entities": [ - ("unit", "coal_plant"), - ("node", "southern_hemisphere"), - ("node", "northern_hemisphere"), - ("model", "all_year_round"), - ("direction", "up"), - ("direction", "down"), + ["unit", "coal_plant"], + ["node", "southern_hemisphere"], + ["node", "northern_hemisphere"], + ["model", "all_year_round"], + ["direction", "up"], + ["direction", "down"], + ], + }, + ) + + def test_import_entities_with_description(self): + header = ["Class", "Entity", "Description"] + data_source = iter( + [ + ["unit", "coal_plant", "Where coal is sacrificed to please the gods of Power."], + ["direction", "up", None], + ["direction", "down", ""], + ] + ) + flattened = default_import_mapping("EntityClass").flatten() + flattened[0].position = 0 + flattened[2].position = 1 + flattened[3].position = 2 + mapped_data, errors = get_mapped_data(data_source, [to_dict(unflatten(flattened))], header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "entity_classes": [ + ["unit", []], + ["direction", []], + ], + "entities": [ + ["unit", "coal_plant", "Where coal is sacrificed to please the gods of Power."], + ["direction", "up"], + ["direction", "down"], + ], + }, + ) + + def test_import_multidimensional_entities_with_descriptions(self): + header = ["Widget", "Gadget", "Description"] + data_source = iter( + [ + ["check_box", "mobile_phone", "A cozy relationship."], + ] + ) + mappings = [ + [ + {"map_type": "EntityClass", "position": "hidden", "value": "Widget__Gadget"}, + {"map_type": "Dimension", "position": "hidden", "value": "Widget"}, + {"map_type": "Dimension", "position": "hidden", "value": "Gadget"}, + {"map_type": "EntityClassDescription", "position": "hidden"}, + {"map_type": "Entity", "position": "hidden", "value": "relationship"}, + {"map_type": "Element", "position": 0, "import_objects": True}, + {"map_type": "Element", "position": 1, "import_objects": True}, + {"map_type": "EntityDescription", "position": 2}, + ] + ] + mapped_data, errors = get_mapped_data(data_source, mappings, header) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "entity_classes": [ + ["Widget__Gadget", ["Widget", "Gadget"]], + ["Widget", []], + ["Gadget", []], + ], + "entities": [ + ["Widget__Gadget", ["check_box", "mobile_phone"], "A cozy relationship."], + ["Widget", "check_box"], + ["Gadget", "mobile_phone"], ], }, ) diff --git a/tests/import_mapping/test_import_mapping.py b/tests/import_mapping/test_import_mapping.py index b8988398..a14df131 100644 --- a/tests/import_mapping/test_import_mapping.py +++ b/tests/import_mapping/test_import_mapping.py @@ -61,7 +61,7 @@ def test_convert_functions_float(self): mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { "entity_classes": [["a", []]], - "entities": [("a", "obj")], + "entities": [["a", "obj"]], "parameter_definitions": [("a", "param", 1.2)], } self.assertEqual(mapped_data, expected) @@ -80,7 +80,7 @@ def test_convert_functions_str(self): mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { "entity_classes": [["a", []]], - "entities": [("a", "obj")], + "entities": [["a", "obj"]], "parameter_definitions": [("a", "param", "1111.2222")], } self.assertEqual(mapped_data, expected) @@ -99,7 +99,7 @@ def test_convert_functions_bool(self): mapped_data, _ = get_mapped_data(data, [mapping], column_convert_fns=column_convert_fns) expected = { "entity_classes": [["a", []]], - "entities": [("a", "obj")], + "entities": [["a", "obj"]], "parameter_definitions": [("a", "param", False)], } self.assertEqual(mapped_data, expected) @@ -814,7 +814,7 @@ def test_read_flat_file(self): ] expected = { "entity_classes": [["oc1", []], ["oc2", []]], - "entities": [("oc1", "obj1"), ("oc2", "obj2")], + "entities": [["oc1", "obj1"], ["oc2", "obj2"]], "parameter_definitions": [("oc1", "parameter_name1"), ("oc2", "parameter_name2")], "parameter_values": [["oc1", ("obj1",), "parameter_name1", 1], ["oc2", ("obj2",), "parameter_name2", 2]], } @@ -841,7 +841,7 @@ def test_read_flat_file_array(self): ] expected = { "entity_classes": [["oc1", []]], - "entities": [("oc1", "obj1")], + "entities": [["oc1", "obj1"]], "parameter_definitions": [("oc1", "parameter_name1")], "parameter_values": [["oc1", ("obj1",), "parameter_name1", Array([1, 2])]], } @@ -868,7 +868,7 @@ def test_read_flat_file_array_with_ed(self): ] expected = { "entity_classes": [["oc1", []]], - "entities": [("oc1", "obj1")], + "entities": [["oc1", "obj1"]], "parameter_definitions": [("oc1", "parameter_name1")], "parameter_values": [["oc1", ("obj1",), "parameter_name1", Array([1, 2])]], } @@ -895,7 +895,7 @@ def test_read_flat_file_array_with_ed(self): def test_read_flat_file_with_column_name_reference(self): input_data = [["object", "parameter", "value"], ["obj1", "parameter_name1", 1], ["obj2", "parameter_name2", 2]] - expected = {"entity_classes": [["object", []]], "entities": [("object", "obj1"), ("object", "obj2")]} + expected = {"entity_classes": [["object", []]], "entities": [["object", "obj1"], ["object", "obj2"]]} data = iter(input_data) data_header = next(data) @@ -910,7 +910,7 @@ def test_read_object_class_from_header_using_string_as_integral_index(self): input_data = [["object_class"], ["obj1"], ["obj2"]] expected = { "entity_classes": [["object_class", []]], - "entities": [("object_class", "obj1"), ("object_class", "obj2")], + "entities": [["object_class", "obj1"], ["object_class", "obj2"]], } data = iter(input_data) @@ -926,7 +926,7 @@ def test_read_object_class_from_header_using_string_as_column_header_name(self): input_data = [["object_class"], ["obj1"], ["obj2"]] expected = { "entity_classes": [["object_class", []]], - "entities": [("object_class", "obj1"), ("object_class", "obj2")], + "entities": [["object_class", "obj1"], ["object_class", "obj2"]], } data = iter(input_data) @@ -944,7 +944,7 @@ def test_read_object_class_from_header_using_string_as_column_header_name(self): def test_read_with_list_of_mappings(self): input_data = [["object", "parameter", "value"], ["obj1", "parameter_name1", 1], ["obj2", "parameter_name2", 2]] - expected = {"entity_classes": [["object", []]], "entities": [("object", "obj1"), ("object", "obj2")]} + expected = {"entity_classes": [["object", []]], "entities": [["object", "obj1"], ["object", "obj2"]]} data = iter(input_data) data_header = next(data) @@ -959,7 +959,7 @@ def test_read_pivoted_parameters_from_header(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { "entity_classes": [["object", []]], - "entities": [("object", "obj1"), ("object", "obj2")], + "entities": [["object", "obj1"], ["object", "obj2"]], "parameter_definitions": [("object", "parameter_name1"), ("object", "parameter_name2")], "parameter_values": [ ["object", ("obj1",), "parameter_name1", 0], @@ -1005,7 +1005,7 @@ def test_read_pivoted_parameters_from_data(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { "entity_classes": [["object", []]], - "entities": [("object", "obj1"), ("object", "obj2")], + "entities": [["object", "obj1"], ["object", "obj2"]], "parameter_definitions": [("object", "parameter_name1"), ("object", "parameter_name2")], "parameter_values": [ ["object", ("obj1",), "parameter_name1", 0], @@ -1039,7 +1039,7 @@ def test_pivoted_value_has_actual_position(self): ] expected = { "entity_classes": [["timeline", []]], - "entities": [("timeline", "obj1"), ("timeline", "obj2")], + "entities": [["timeline", "obj1"], ["timeline", "obj2"]], "parameter_definitions": [("timeline", "value")], "alternatives": {"Base"}, "parameter_values": [ @@ -1069,7 +1069,7 @@ def test_import_objects_from_pivoted_data_when_they_lack_parameter_values(self): input_data = [["object", "is_skilled", "has_powers"], ["obj1", "yes", "no"], ["obj2", None, None]] expected = { "entity_classes": [["node", []]], - "entities": [("node", "obj1"), ("node", "obj2")], + "entities": [["node", "obj1"], ["node", "obj2"]], "parameter_definitions": [("node", "is_skilled"), ("node", "has_powers")], "alternatives": {"Base"}, "parameter_values": [ @@ -1100,7 +1100,7 @@ def test_import_objects_from_pivoted_data_when_they_lack_map_type_parameter_valu ] expected = { "entity_classes": [["node", []]], - "entities": [("node", "obj1")], + "entities": [["node", "obj1"]], "parameter_definitions": [("node", "is_skilled"), ("node", "has_powers")], "alternatives": {"Base"}, "parameter_values": [ @@ -1135,7 +1135,7 @@ def test_read_flat_file_with_extra_value_dimensions(self): expected = { "entity_classes": [["object", []]], - "entities": [("object", "obj1")], + "entities": [["object", "obj1"]], "parameter_definitions": [("object", "parameter_name1")], "parameter_values": [ [ @@ -1172,7 +1172,7 @@ def test_read_flat_file_with_parameter_definition(self): expected = { "entity_classes": [["object", []]], - "entities": [("object", "obj1")], + "entities": [["object", "obj1"]], "parameter_definitions": [("object", "parameter_name1")], } @@ -1199,7 +1199,7 @@ def test_read_1dim_relationships(self): input_data = [["unit", "node"], ["u1", "n1"], ["u1", "n2"]] expected = { "entity_classes": [["node_group", ["node"]]], - "entities": [("node_group", ["n1"]), ("node_group", ["n2"])], + "entities": [["node_group", ["n1"]], ["node_group", ["n2"]]], } data = iter(input_data) @@ -1220,7 +1220,7 @@ def test_read_relationships(self): input_data = [["unit", "node"], ["u1", "n1"], ["u1", "n2"]] expected = { "entity_classes": [["unit__node", ["unit", "node"]]], - "entities": [("unit__node", ["u1", "n1"]), ("unit__node", ["u1", "n2"])], + "entities": [["unit__node", ["u1", "n1"]], ["unit__node", ["u1", "n2"]]], } data = iter(input_data) @@ -1244,7 +1244,7 @@ def test_read_relationships_with_parameters(self): input_data = [["unit", "node", "rel_parameter"], ["u1", "n1", 0], ["u1", "n2", 1]] expected = { "entity_classes": [["unit__node", ["unit", "node"]]], - "entities": [("unit__node", ["u1", "n1"]), ("unit__node", ["u1", "n2"])], + "entities": [["unit__node", ["u1", "n1"]], ["unit__node", ["u1", "n2"]]], "parameter_definitions": [("unit__node", "rel_parameter")], "parameter_values": [ ["unit__node", ("u1", "n1"), "rel_parameter", 0], @@ -1275,11 +1275,11 @@ def test_read_relationships_with_parameters2(self): expected = { "entity_classes": [["nuts2__fueltype", ["nuts2", "fueltype"]], ["nuts2", []], ["fueltype", []]], "entities": [ - ("nuts2__fueltype", ["BE23", "Bioenergy"]), - ("nuts2", "BE23"), - ("fueltype", "Bioenergy"), - ("nuts2__fueltype", ["DE11", "Bioenergy"]), - ("nuts2", "DE11"), + ["nuts2__fueltype", ["BE23", "Bioenergy"]], + ["nuts2", "BE23"], + ["fueltype", "Bioenergy"], + ["nuts2__fueltype", ["DE11", "Bioenergy"]], + ["nuts2", "DE11"], ], "parameter_definitions": [("nuts2__fueltype", "capacity")], "parameter_values": [ @@ -1318,7 +1318,7 @@ def test_read_parameter_header_with_only_one_parameter(self): input_data = [["object", "parameter_name1"], ["obj1", 0], ["obj2", 2]] expected = { "entity_classes": [["object", []]], - "entities": [("object", "obj1"), ("object", "obj2")], + "entities": [["object", "obj1"], ["object", "obj2"]], "parameter_definitions": [("object", "parameter_name1")], "parameter_values": [ ["object", ("obj1",), "parameter_name1", 0], @@ -1344,7 +1344,7 @@ def test_read_pivoted_parameters_from_data_with_skipped_column(self): input_data = [["object", "parameter_name1", "parameter_name2"], ["obj1", 0, 1], ["obj2", 2, 3]] expected = { "entity_classes": [["object", []]], - "entities": [("object", "obj1"), ("object", "obj2")], + "entities": [["object", "obj1"], ["object", "obj2"]], "parameter_definitions": [("object", "parameter_name1")], "parameter_values": [ ["object", ("obj1",), "parameter_name1", 0], @@ -1375,12 +1375,12 @@ def test_read_relationships_and_import_objects(self): ["node", []], ], "entities": [ - ("unit__node", ["u1", "n1"]), - ("unit", "u1"), - ("node", "n1"), - ("unit__node", ["u2", "n2"]), - ("unit", "u2"), - ("node", "n2"), + ["unit__node", ["u1", "n1"]], + ["unit", "u1"], + ["node", "n1"], + ["unit__node", ["u2", "n2"]], + ["unit", "u2"], + ["node", "n2"], ], } @@ -1407,7 +1407,7 @@ def test_read_relationships_parameter_values_with_extra_dimensions(self): expected = { "entity_classes": [["unit__node", ["unit", "node"]]], "parameter_definitions": [("unit__node", "e"), ("unit__node", "f")], - "entities": [("unit__node", ["a", "c"]), ("unit__node", ["b", "d"])], + "entities": [["unit__node", ["a", "c"]], ["unit__node", ["b", "d"]]], "parameter_values": [ ["unit__node", ("a", "c"), "e", Map(["a", "b"], [2, 4])], ["unit__node", ("b", "d"), "f", Map(["a", "b"], [3, 5])], @@ -1443,7 +1443,7 @@ def test_read_data_with_read_start_row(self): ] expected = { "entity_classes": [["oc1", []], ["oc2", []]], - "entities": [("oc1", "obj1"), ("oc2", "obj2")], + "entities": [["oc1", "obj1"], ["oc2", "obj2"]], "parameter_definitions": [("oc1", "parameter_name1"), ("oc2", "parameter_name2")], "parameter_values": [["oc1", ("obj1",), "parameter_name1", 1], ["oc2", ("obj2",), "parameter_name2", 2]], } @@ -1472,7 +1472,7 @@ def test_read_data_with_two_mappings_with_different_read_start_row(self): ] expected = { "entity_classes": [["oc1", []], ["oc2", []]], - "entities": [("oc1", "oc1_obj1"), ("oc1", "oc1_obj2"), ("oc2", "oc2_obj2")], + "entities": [["oc1", "oc1_obj1"], ["oc1", "oc1_obj2"], ["oc2", "oc2_obj2"]], "parameter_definitions": [("oc1", "parameter_class1"), ("oc2", "parameter_class2")], "parameter_values": [ ["oc1", ("oc1_obj1",), "parameter_class1", 1], @@ -1515,7 +1515,7 @@ def test_read_object_class_with_table_name_as_class_name(self): out, errors = get_mapped_data(data, [mapping], data_header, "class name") expected = { "entity_classes": [["class name", []]], - "entities": [("class name", "object 1"), ("class name", "object 2")], + "entities": [["class name", "object 1"], ["class name", "object 2"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1540,7 +1540,7 @@ def test_read_flat_map_from_columns(self): expected_map = Map(["key1", "key2"], [-2, -1]) expected = { "entity_classes": [["object_class", []]], - "entities": [("object_class", "object")], + "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } @@ -1567,7 +1567,7 @@ def test_read_nested_map_from_columns(self): expected_map = Map(["key11", "key21"], [Map(["key12"], [-2]), Map(["key22"], [-1])]) expected = { "entity_classes": [["object_class", []]], - "entities": [("object_class", "object")], + "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } @@ -1611,7 +1611,7 @@ def test_read_uneven_nested_map_from_columns(self): ) expected = { "entity_classes": [["object_class", []]], - "entities": [("object_class", "object")], + "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } @@ -1653,7 +1653,7 @@ def test_read_nested_map_with_compression(self): ) expected = { "entity_classes": [["object_class", []]], - "entities": [("object_class", "object")], + "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } @@ -1834,11 +1834,11 @@ def test_read_object_group_and_import_objects(self): }, "entity_classes": [["class_A", []]], "entities": [ - ("class_A", "group1"), - ("class_A", "object1"), - ("class_A", "object2"), - ("class_A", "group2"), - ("class_A", "object3"), + ["class_A", "group1"], + ["class_A", "object1"], + ["class_A", "object2"], + ["class_A", "group2"], + ["class_A", "object3"], ], } self.assertFalse(errors) @@ -1943,7 +1943,7 @@ def test_read_map_index_names_from_columns(self): ) expected = { "entity_classes": [["object_class", []]], - "entities": [("object_class", "object")], + "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } @@ -1975,7 +1975,7 @@ def test_missing_map_index_name(self): ) expected = { "entity_classes": [["object_class", []]], - "entities": [("object_class", "object")], + "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], "parameter_definitions": [("object_class", "parameter")], } @@ -2016,7 +2016,7 @@ def test_filter_regular_expression_in_root_mapping(self): data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0, filter_re="B"), EntityMapping(1)]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [["B", []]], "entities": [("B", "r")]} + expected = {"entity_classes": [["B", []]], "entities": [["B", "r"]]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2025,7 +2025,7 @@ def test_filter_regular_expression_in_child_mapping(self): data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0), EntityMapping(1, filter_re="q|r")]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [["A", []], ["B", []]], "entities": [("A", "q"), ("B", "r")]} + expected = {"entity_classes": [["A", []], ["B", []]], "entities": [["A", "q"], ["B", "r"]]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2034,7 +2034,7 @@ def test_filter_regular_expression_in_child_mapping_filters_parent_mappings_too( data = iter(input_data) mapping_root = unflatten([EntityClassMapping(0), EntityMapping(1, filter_re="q")]) out, errors = get_mapped_data(data, [mapping_root]) - expected = {"entity_classes": [["A", []]], "entities": [("A", "q")]} + expected = {"entity_classes": [["A", []]], "entities": [["A", "q"]]} self.assertFalse(errors) self.assertEqual(out, expected) @@ -2054,7 +2054,7 @@ def test_arrays_get_imported_to_correct_alternatives(self): out, errors = get_mapped_data(data, [mapping_root]) expected = { "entity_classes": [["class", []]], - "entities": [("class", "y")], + "entities": [["class", "y"]], "parameter_definitions": [("class", "parameter")], "alternatives": {"Base", "alternative"}, "parameter_values": [ diff --git a/tests/spine_io/importers/test_reader.py b/tests/spine_io/importers/test_reader.py index 5d158eed..d2243a8b 100644 --- a/tests/spine_io/importers/test_reader.py +++ b/tests/spine_io/importers/test_reader.py @@ -58,7 +58,7 @@ def test_get_mapped_data(self): table_row_convert_specs, ) self.assertEqual(errors, []) - self.assertEqual(mapped_data, {"entity_classes": [["A", []]], "entities": [("A", "b")]}) + self.assertEqual(mapped_data, {"entity_classes": [["A", []]], "entities": [["A", "b"]]}) def test_resolve_values_for_fixed_position_mappings(self): reader = Reader(None) From c2b15f54eaa692c6bcd5fd8c2a5498d79910ad6e Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Mon, 2 Mar 2026 09:17:17 +0100 Subject: [PATCH 5/9] Add parameter description mapping to import mappings Re spine-tools/Spine-Toolbox#1892 --- spinedb_api/import_mapping/generator.py | 161 ++--- spinedb_api/import_mapping/import_mapping.py | 629 +++++++++++-------- tests/import_mapping/test_generator.py | 41 +- tests/import_mapping/test_import_mapping.py | 119 ++-- 4 files changed, 566 insertions(+), 384 deletions(-) diff --git a/spinedb_api/import_mapping/generator.py b/spinedb_api/import_mapping/generator.py index 29922b49..1abe6702 100644 --- a/spinedb_api/import_mapping/generator.py +++ b/spinedb_api/import_mapping/generator.py @@ -16,12 +16,15 @@ """ from collections.abc import Callable from copy import deepcopy +from itertools import dropwhile from typing import Any, Optional from ..exception import ParameterValueFormatError from ..helpers import string_to_bool +from ..import_functions import UnparseCallable from ..mapping import Position, is_pivoted from ..parameter_value import ( Array, + IndexedValue, Map, TimePattern, TimeSeriesVariableResolution, @@ -29,7 +32,16 @@ from_database, split_value_and_type, ) -from .import_mapping import ImportMapping, check_validity +from .import_mapping import ( + ArrayValueRecord, + ImportMapping, + MapValueRecord, + SemiMappedData, + TimePatternValueRecord, + TimeSeriesValueRecord, + ValueRecord, + check_validity, +) from .import_mapping_compat import import_mapping_from_dict _NO_VALUE = object() @@ -177,6 +189,7 @@ def get_mapped_data( _make_entities(mapped_data) _make_entity_metadata(mapped_data) _make_entity_alternatives(mapped_data, errors) + _make_parameter_definitions(mapped_data, unparse_value) _make_parameter_values(mapped_data, unparse_value) _make_parameter_value_metadata(mapped_data) return mapped_data, errors @@ -342,35 +355,59 @@ def _make_entity_alternatives(mapped_data, errors): mapped_data["entity_alternatives"] = rows +def _make_parameter_definitions(mapped_data: SemiMappedData, unparse_value: UnparseCallable) -> None: + key = "parameter_definitions" + try: + rows = mapped_data.pop(key) + except KeyError: + return + final_rows = [] + for (entity_class_name, parameter_name), record in rows.items(): + definition_data = [entity_class_name, parameter_name] + default_value = record.default_value + if isinstance(default_value, ValueRecord): + if default_value.has_value(): + default_value = unparse_value(_make_value(default_value)) + else: + default_value = None + elif isinstance(default_value, str): + try: + default_value = from_database(*split_value_and_type(default_value)) + except ParameterValueFormatError: + pass + reversed_extras = [record.description, record.value_list_name, default_value] + definition_data += reversed(list(dropwhile(lambda x: x is None, reversed_extras))) + final_rows.append(definition_data) + if final_rows: + mapped_data[key] = final_rows + + def _make_parameter_values(mapped_data, unparse_value): - value_pos = 3 key = "parameter_values" - rows = mapped_data.get(key) - if rows is not None: - valued_rows = [] - for row in rows: - raw_value = _make_value(row, value_pos) - if raw_value is _NO_VALUE: - continue - value = unparse_value(raw_value) - if value is not None: - row[value_pos] = value - valued_rows.append(row) - mapped_data[key] = valued_rows - value_pos = 0 - key = "parameter_definitions" - rows = mapped_data.get(key) - if rows is not None: - full_rows = [] - for entity_definition, extras in rows.items(): - if extras: - value = unparse_value(_make_value(extras, value_pos)) - if value is not None: - extras[value_pos] = value - full_rows.append(entity_definition + tuple(extras)) + try: + rows = mapped_data.pop(key) + except KeyError: + return + final_rows = [] + for (entity_class_name, entity_byname, parameter_name, alternative_name), value in rows.items(): + if isinstance(value, ValueRecord): + if value.has_value(): + value = unparse_value(_make_value(value)) else: - full_rows.append(entity_definition) - mapped_data[key] = full_rows + value = None + elif isinstance(value, str): + try: + value = from_database(*split_value_and_type(value)) + except ParameterValueFormatError: + pass + if value is None: + continue + value_data = [entity_class_name, entity_byname, parameter_name, value] + if alternative_name is not None: + value_data.append(alternative_name) + final_rows.append(value_data) + if final_rows: + mapped_data[key] = final_rows def _make_parameter_value_metadata(mapped_data): @@ -387,42 +424,28 @@ def _make_entity_metadata(mapped_data): mapped_data["entity_metadata"] = list(rows) -def _make_value(row, value_pos): - try: - value = row[value_pos] - except IndexError: - return None - if isinstance(value, dict): - if "data" not in value: - return _NO_VALUE - return _parameter_value_from_dict(value) - if isinstance(value, str): - try: - return from_database(*split_value_and_type(value)) - except ParameterValueFormatError: - pass - return value - - -def _parameter_value_from_dict(d): - mapped_index_names = d.get("index_names", {0: ""}) - index_names = (max(mapped_index_names) + 1) * [""] - for i, name in mapped_index_names.items(): - index_names[i] = name - if d["type"] == "map": - map_ = _table_to_map(d["data"], compress=d.get("compress", False)) - if index_names != [""]: - _apply_index_names(map_, index_names) - return map_ - if d["type"] == "time_pattern": - return TimePattern(*zip(*d["data"]), index_name=index_names[0]) - if d["type"] == "time_series": - options = d.get("options", {}) - ignore_year = options.get("ignore_year", False) - repeat = options.get("repeat", False) - return TimeSeriesVariableResolution(*zip(*d["data"]), ignore_year, repeat, index_name=index_names[0]) - if d["type"] == "array": - return Array(d["data"], index_name=index_names[0]) +def _make_value(record: ValueRecord) -> IndexedValue: + match record: + case ArrayValueRecord(): + index_name = record.index_names[0] if record.index_names else "" + return Array(record.values, index_name=index_name) + case TimePatternValueRecord(): + index_name = record.index_names[0] if record.index_names else "" + indexes = [i[0] for i in record.indexes] + return TimePattern(indexes, record.values, index_name) + case TimeSeriesValueRecord(): + index_name = record.index_names[0] if record.index_names else "" + indexes = [i[0] for i in record.indexes] + return TimeSeriesVariableResolution(indexes, record.values, record.ignore_year, record.repeat, index_name) + case MapValueRecord(): + map_value = _table_to_map( + ([*indexes, values] for indexes, values in zip(record.indexes, record.values)), record.compress + ) + if record.index_names: + _apply_index_names(map_value, record.index_names) + return map_value + case _: + raise RuntimeError(f"logic error: unknown record type '{type(record).__name__}'") def _table_to_map(table, compress=False): @@ -466,7 +489,7 @@ def _apply_index_names(map_, index_names): """ name = index_names[0] if name: - map_.index_name = index_names[0] + map_.index_name = name if len(index_names) == 1: return for v in map_.values: @@ -474,16 +497,14 @@ def _apply_index_names(map_, index_names): _apply_index_names(v, index_names[1:]) -def _ensure_mapping_name_consistency(mappings, mapping_names): +def _ensure_mapping_name_consistency(mappings: list[ImportMapping], mapping_names: list[str]) -> None: """Makes sure that there are as many mapping names as actual mappings. Args: - mappings (list(ImportMapping)): list of mappings - mapping_names (list(str)): list of mapping names + mappings: list of mappings + mapping_names: list of mapping names """ n_mappings = len(mappings) n_mapping_names = len(mapping_names) - if n_mapping_names > n_mappings: - mapping_names = mapping_names[:n_mappings] - elif n_mapping_names < n_mappings: + if n_mapping_names < n_mappings: mapping_names += [""] * (n_mappings - n_mapping_names) diff --git a/spinedb_api/import_mapping/import_mapping.py b/spinedb_api/import_mapping/import_mapping.py index 19f5fd73..37c31935 100644 --- a/spinedb_api/import_mapping/import_mapping.py +++ b/spinedb_api/import_mapping/import_mapping.py @@ -10,10 +10,11 @@ # this program. If not, see . ###################################################################################################################### """Contains import mappings for database items such as entities, entity classes and parameter values.""" +from __future__ import annotations from collections.abc import Iterable from dataclasses import dataclass, field from enum import Enum, auto, unique -from typing import Any, ClassVar, TypeAlias +from typing import Any, ClassVar, Generic, Type, TypeAlias, TypeVar from spinedb_api.exception import InvalidMapping, InvalidMappingComponent from spinedb_api.mapping import Mapping, Position, is_pivoted, parse_fixed_position_value, unflatten @@ -28,12 +29,12 @@ class ImportKey(Enum): METADATA_NAME = auto() METADATA_VALUE = auto() PARAMETER_NAME = auto() - PARAMETER_DEFINITION = auto() - PARAMETER_DEFINITION_EXTRAS = auto() - PARAMETER_DEFAULT_VALUES = auto() + PARAMETER_DEFAULT_VALUE_RECORD = auto() PARAMETER_DEFAULT_VALUE_INDEXES = auto() - PARAMETER_VALUES = auto() + PARAMETER_DEFAULT_VALUE_INDEX_NAMES = auto() + PARAMETER_VALUE_RECORD = auto() PARAMETER_VALUE_INDEXES = auto() + PARAMETER_VALUE_INDEX_NAMES = auto() PARAMETER_VALUE_METADATA_NAME = auto() PARAMETER_VALUE_METADATA_VALUE = auto() ALTERNATIVE_NAME = auto() @@ -53,7 +54,6 @@ def __str__(self): self.METADATA_NAME: "Metadata names", self.METADATA_VALUE: "Metadata values", self.PARAMETER_NAME.value: "Parameter names", - self.PARAMETER_DEFINITION.value: "Parameter names", self.PARAMETER_DEFAULT_VALUE_INDEXES.value: "Parameter indexes", self.PARAMETER_VALUE_INDEXES.value: "Parameter indexes", self.PARAMETER_VALUE_METADATA_NAME.value: "Metadata names", @@ -73,21 +73,19 @@ def __str__(self): SemiMappedData: TypeAlias = dict[str, Any] -def check_validity(root_mapping): - class _DummySourceRow: - def __getitem__(self, key): - return "true" - +def check_validity(root_mapping: ImportMapping) -> list[InvalidMappingComponent]: errors = [] for rank, mapping in enumerate(root_mapping.flatten()): - if mapping.position != Position.fixed: - continue - try: - parse_fixed_position_value(mapping.value) - except InvalidMapping as error: - errors.append(InvalidMappingComponent(str(error), rank)) - source_row = _DummySourceRow() - root_mapping.import_row(source_row, {}, {}, errors) + if mapping.position == Position.fixed: + try: + parse_fixed_position_value(mapping.value) + except InvalidMapping as error: + errors.append(InvalidMappingComponent(str(error), rank)) + elif mapping.position != Position.hidden or mapping.value is not None: + try: + mapping.check_validity() + except InvalidMappingComponent as error: + errors.append(error) return errors @@ -177,6 +175,9 @@ def check_for_invalid_column_refs(self, header, table_name): return msg return "" + def check_validity(self) -> None: + return + def polish(self, table_name, source_header, mapping_name, column_count=0, for_preview=False): """Polishes the mapping before an import operation. 'Expands' transient ``position`` and ``value`` attributes into their final value. @@ -396,6 +397,21 @@ def reconstruct(cls, position, value, skip_columns, read_start_row, filter_re, m mapping = cls(position, value, skip_columns, read_start_row, filter_re, compress, options) return mapping + def _make_value_record(self, value_type: str) -> ValueRecord: + match value_type: + case "array": + return ArrayValueRecord() + case "map": + return MapValueRecord(compress=self.compress) + case "time_series": + return TimeSeriesValueRecord( + ignore_year=self.options.get("ignore_year", False), repeat=self.options.get("repeat", False) + ) + case "time_pattern": + return TimePatternValueRecord() + case _: + raise InvalidMapping(f"unknown value type '{value_type}'") + @dataclass class EntityClassRecord: @@ -404,10 +420,7 @@ class EntityClassRecord: class EntityClassMapping(ImportMapping): - """Maps entity classes. - - Can be used as the topmost mapping. - """ + """Maps entity classes.""" MAP_TYPE = "EntityClass" @@ -417,11 +430,51 @@ def _import_row(self, source_data, state, mapped_data): entity_classes[entity_class_name] = EntityClassRecord() -class EntityClassDescriptionMapping(ImportMapping): - """Maps entity class descriptions. +def _require_parent(mapping: ImportMapping, parent_type: Type[ImportMapping]) -> None: + parent = mapping.parent + while parent is not None: + if isinstance(parent, parent_type): + if parent.position == Position.hidden and parent.value is None: + raise InvalidMappingComponent( + f"{mapping.MAP_TYPE} requires {parent_type.MAP_TYPE} with position or constant value", mapping.rank + ) + return + parent = parent.parent + raise InvalidMappingComponent(f"{mapping.MAP_TYPE} requires {parent_type.MAP_TYPE} as parent", mapping.rank) - Cannot be used as the topmost mapping; one of the parents must be :class:`EntityClassMapping`. - """ + +def _require_one_of_parents(mapping: ImportMapping, parent_types: tuple[Type[ImportMapping], ...]) -> None: + parent = mapping.parent + while parent is not None: + if isinstance(parent, parent_types) and (parent.position != Position.hidden or parent.value is not None): + return + parent = parent.parent + display_types = " or ".join(m.MAP_TYPE for m in parent_types) + raise InvalidMappingComponent(f"{mapping.MAP_TYPE} requires {display_types} as parent", mapping.rank) + + +def _require_enough_parents(mapping: ImportMapping, parent_type: Type[ImportMapping]) -> None: + n_same_parent_type = 1 + parent = mapping.parent + while parent is not None: + if isinstance(parent, type(mapping)): + n_same_parent_type += 1 + parent = parent.parent + n_required_parent_type = 0 + parent = mapping.parent + while parent is not None: + if isinstance(parent, parent_type): + n_required_parent_type += 1 + if n_required_parent_type == n_same_parent_type: + return + parent = parent.parent + raise InvalidMappingComponent( + f"the number of {mapping.MAP_TYPE} and {parent_type.MAP_TYPE} mappings do not match", mapping.rank + ) + + +class EntityClassDescriptionMapping(ImportMapping): + """Maps entity class descriptions.""" MAP_TYPE = "EntityClassDescription" ignorable = True @@ -432,6 +485,9 @@ def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] mapped_data["entity_classes"][entity_class_name].description = description + def check_validity(self) -> None: + _require_parent(self, EntityClassMapping) + @dataclass class EntityRecord: @@ -440,10 +496,7 @@ class EntityRecord: class EntityMapping(ImportMapping): - """Maps entities. - - Cannot be used as the topmost mapping; one of the parents must be :class:`EntityClassMapping`. - """ + """Maps entities.""" MAP_TYPE = "Entity" @@ -454,12 +507,12 @@ def _import_row(self, source_data, state, mapped_data): entity_name = state[ImportKey.ENTITY_NAME] = str(source_data) mapped_data.setdefault("entities", {})[entity_class_name, entity_name] = EntityRecord() + def check_validity(self) -> None: + _require_parent(self, EntityClassMapping) -class EntityDescriptionMapping(ImportMapping): - """Maps entity descriptions. - Cannot be used as the topmost mapping; one of the parents must be :class:`EntityMapping` or :class:`ElementMapping`. - """ +class EntityDescriptionMapping(ImportMapping): + """Maps entity descriptions.""" MAP_TYPE = "EntityDescription" ignorable = True @@ -471,6 +524,9 @@ def _import_row(self, source_data, state, mapped_data): entity_name = state[ImportKey.ENTITY_NAME] mapped_data["entities"][entity_class_name, entity_name].description = description + def check_validity(self) -> None: + _require_parent(self, EntityMapping) + class EntityMetadataNameMapping(ImportMapping): """Maps entity metadata names.""" @@ -483,10 +539,7 @@ def _import_row(self, source_data, state, mapped_data): class EntityMetadataValueMapping(ImportMapping): - """Maps entity metadata names. - - Cannot be used as the topmost mapping; must have :class:`EntityClassMapping`, :class:`EntityMapping` and :class:`EntityMetadataValueMapping` as parent. - """ + """Maps entity metadata names.""" MAP_TYPE = "EntityMetadataValue" ignorable = True @@ -500,12 +553,12 @@ def _import_row(self, source_data, state, mapped_data): entity_class_name, entity_byname, metadata_name, metadata_value ] = None + def check_validity(self) -> None: + _require_parent(self, EntityMetadataNameMapping) -class EntityGroupMapping(ImportEntitiesMixin, ImportMapping): - """Maps entity groups. - Cannot be used as the topmost mapping; must have :class:`EntityClassMapping` and :class:`EntityMapping` as parents. - """ +class EntityGroupMapping(ImportEntitiesMixin, ImportMapping): + """Maps entity groups.""" MAP_TYPE = "EntityGroup" @@ -522,13 +575,12 @@ def _import_row(self, source_data, state, mapped_data): except KeyError: pass + def check_validity(self) -> None: + _require_parent(self, EntityMapping) -class EntityAlternativeActivityMapping(ImportMapping): - """Maps activity flags for entity alternative. - Cannot be used as the topmost mapping; must have :class:`EntityMapping` or :class:`ElementMapping`, - and :class:`AlternativeMapping` as parents. - """ +class EntityAlternativeActivityMapping(ImportMapping): + """Maps activity flags for entity alternative.""" MAP_TYPE = "EntityAlternativeActivity" ignorable = True @@ -543,12 +595,13 @@ def _import_row(self, source_data, state, mapped_data): entity_class_name, entity_byname, alternative_name, source_data ] = None + def check_validity(self) -> None: + _require_parent(self, EntityMapping) + _require_parent(self, AlternativeMapping) -class DimensionMapping(ImportMapping): - """Maps dimensions. - Cannot be used as the topmost mapping; one of the parents must be :class:`EntityClassMapping`. - """ +class DimensionMapping(ImportMapping): + """Maps dimensions.""" MAP_TYPE = "Dimension" @@ -557,13 +610,12 @@ def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] mapped_data["entity_classes"][entity_class_name].dimensions.append(dimension_name) + def check_validity(self) -> None: + _require_parent(self, EntityClassMapping) -class ElementMapping(ImportEntitiesMixin, ImportMapping): - """Maps elements. - Cannot be used as the topmost mapping; must have :class:`DimensionMapping` and :class:`EntityMapping` - as parents. - """ +class ElementMapping(ImportEntitiesMixin, ImportMapping): + """Maps elements.""" MAP_TYPE = "Element" @@ -596,6 +648,9 @@ def _import_row(self, source_data, state, mapped_data): if (dimension_name, element_name) not in mapped_entities: mapped_entities[dimension_name, element_name] = EntityRecord() + def check_validity(self) -> None: + _require_enough_parents(self, DimensionMapping) + class MetadataNameMapping(ImportMapping): """Maps metadata names.""" @@ -607,10 +662,7 @@ def _import_row(self, source_data, state, mapped_data): class MetadataValueMapping(ImportMapping): - """Maps metadata values. - - Cannot be used as the topmost mapping; must have a metadata name mapping as one of parents. - """ + """Maps metadata values.""" MAP_TYPE = "MetadataValue" @@ -619,30 +671,88 @@ def _import_row(self, source_data, state, mapped_data): metadata_value = state[ImportKey.METADATA_VALUE] = str(source_data) mapped_data.setdefault("metadata", []).append((metadata_name, metadata_value)) + def check_validity(self) -> None: + _require_parent(self, MetadataNameMapping) -class ParameterDefinitionMapping(ImportMapping): - """Maps parameter definitions. - Cannot be used as the topmost mapping; must have an entity class mapping as one of parents. - """ +T = TypeVar("T") + + +@dataclass +class ValueRecord(Generic[T]): + index_names: list[str] = field(default_factory=list) + indexes: list[list] = field(default_factory=list) + values: list[T] = field(default_factory=list) + + def has_value(self) -> bool: + return bool(self.values) + + +@dataclass +class ArrayValueRecord(ValueRecord[Any]): + pass + + +@dataclass +class TimePatternValueRecord(ValueRecord[float]): + pass + + +@dataclass +class MapValueRecord(ValueRecord[Any]): + compress: bool = False + + +@dataclass +class TimeSeriesValueRecord(ValueRecord[float]): + ignore_year: bool = False + repeat: bool = False + indexes: list = field(default_factory=list) + + +@dataclass +class ParameterDefinitionRecord: + value_list_name: str | None = None + default_value: ValueRecord | None = None + description: str | None = None + + +class ParameterDefinitionMapping(ImportMapping): + """Maps parameter definitions.""" MAP_TYPE = "ParameterDefinition" def _import_row(self, source_data, state, mapped_data): entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) parameter_name = state[ImportKey.PARAMETER_NAME] = str(source_data) - definition_extras = state[ImportKey.PARAMETER_DEFINITION_EXTRAS] = [] - parameter_definition_key = state[ImportKey.PARAMETER_DEFINITION] = entity_class_name, parameter_name - default_values = state.get(ImportKey.PARAMETER_DEFAULT_VALUES) - if default_values is None or parameter_definition_key not in default_values: - mapped_data.setdefault("parameter_definitions", {})[parameter_definition_key] = definition_extras + parameter_definition_key = entity_class_name, parameter_name + definitions = mapped_data.setdefault("parameter_definitions", {}) + if parameter_definition_key not in definitions: + definitions[parameter_definition_key] = ParameterDefinitionRecord() + def check_validity(self) -> None: + _require_parent(self, EntityClassMapping) -class ParameterTypeMapping(ImportMapping): - """Maps parameter types. - Cannot be used as the topmost mapping; must have a parameter definition mapping as one of parents. - """ +class ParameterDefinitionDescriptionMapping(ImportMapping): + """Maps parameter definition descriptions.""" + + MAP_TYPE = "ParameterDefinitionDescription" + ignorable = True + + def _import_row(self, source_data, state, mapped_data): + description = str(source_data) + if description: + entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) + parameter_name = state[ImportKey.PARAMETER_NAME] + mapped_data["parameter_definitions"][entity_class_name, parameter_name].description = description + + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) + + +class ParameterTypeMapping(ImportMapping): + """Maps parameter types.""" MAP_TYPE = "ParameterType" @@ -654,12 +764,12 @@ def _import_row(self, source_data, state, mapped_data): parameter = state[ImportKey.PARAMETER_NAME] mapped_data.setdefault("parameter_types", []).append((entity_class, parameter, parameter_type)) + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) -class ParameterDefaultValueMapping(ImportMapping): - """Maps scalar (non-indexed) default values - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping` as parent. - """ +class ParameterDefaultValueMapping(ImportMapping): + """Maps scalar (non-indexed) default values.""" MAP_TYPE = "ParameterDefaultValue" @@ -667,91 +777,62 @@ def _import_row(self, source_data, state, mapped_data): default_value = source_data if default_value == "": return - parameter_definition_extras = state[ImportKey.PARAMETER_DEFINITION_EXTRAS] - parameter_definition_extras.append(default_value) - value_list_name = state.get(ImportKey.PARAMETER_VALUE_LIST_NAME) - if value_list_name is not None: - parameter_definition_extras.append(value_list_name) + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + parameter_name = state[ImportKey.PARAMETER_NAME] + mapped_data["parameter_definitions"][entity_class_name, parameter_name].default_value = default_value + + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) class ParameterDefaultValueTypeMapping(IndexedValueMixin, ImportMapping): + """Maps indexed default values.""" + MAP_TYPE = "ParameterDefaultValueType" def _import_row(self, source_data, state, mapped_data): - parameter_definition = state.get(ImportKey.PARAMETER_DEFINITION) - if parameter_definition is None: - # Don't catch errors here, this one's invisible - return - default_values = state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUES, {}) - if parameter_definition in default_values: + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + parameter_name = state[ImportKey.PARAMETER_NAME] + key = (entity_class_name, parameter_name) + definition_record = mapped_data["parameter_definitions"][key] + if definition_record.default_value is not None: + state[ImportKey.PARAMETER_DEFAULT_VALUE_RECORD] = definition_record.default_value return - value_type = str(source_data) - default_value = default_values[parameter_definition] = {"type": value_type} - if self.compress and value_type == "map": - default_value["compress"] = self.compress - if self.options and value_type == "time_series": - default_value["options"] = self.options - parameter_definition_extras = state[ImportKey.PARAMETER_DEFINITION_EXTRAS] - parameter_definition_extras.append(default_value) - value_list_name = state.get(ImportKey.PARAMETER_VALUE_LIST_NAME) - if value_list_name is not None: - parameter_definition_extras.append(value_list_name) - - -class IndexNameMappingBase(ImportMapping): - """Base class for index name mappings.""" - - _STATE_KEY = NotImplemented - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._id = None - - @staticmethod - def _value_key(state: State, mapped_data: SemiMappedData) -> tuple: - raise NotImplementedError() + record = self._make_value_record(source_data) + definition_record.default_value = record + state[ImportKey.PARAMETER_DEFAULT_VALUE_RECORD] = record - def _import_row(self, source_data, state, mapped_data): - values = state[self._STATE_KEY] - value = values[self._value_key(state, mapped_data)] - if self._id is None: - self._id = 0 - current = self - while True: - if current.parent is None: - break - current = current.parent - if isinstance(current, type(self)): - self._id += 1 - value.setdefault("index_names", {})[self._id] = source_data - - -class DefaultValueIndexNameMapping(IndexNameMappingBase): - """Maps default value index names. - - Cannot be used as the topmost mapping; must have a :class:`ParameterDefaultValueTypeMapping` as parent. - """ + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) + + +class DefaultValueIndexNameMapping(ImportMapping): + """Maps default value index names.""" MAP_TYPE = "DefaultValueIndexName" - _STATE_KEY = ImportKey.PARAMETER_DEFAULT_VALUES - @staticmethod - def _value_key(state, mapped_data): - return _default_value_key(state) + def _import_row(self, source_data, state, mapped_data): + if ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES in state: + i = len(state[ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES]) + state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUE_INDEX_NAMES, {})[i] = str(source_data) + else: + state[ImportKey.PARAMETER_DEFAULT_VALUE_INDEX_NAMES] = {0: str(source_data)} + def check_validity(self) -> None: + _require_parent(self, ParameterDefaultValueTypeMapping) -class ParameterDefaultValueIndexMapping(ImportMapping): - """Maps default value indexes. - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping` as parent. - """ +class ParameterDefaultValueIndexMapping(ImportMapping): + """Maps default value indexes.""" MAP_TYPE = "ParameterDefaultValueIndex" def _import_row(self, source_data, state, mapped_data): - _ = state[ImportKey.PARAMETER_NAME] - index = source_data - state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES, []).append(index) + state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES, []).append(source_data) + + def check_validity(self) -> None: + _require_parent(self, ParameterDefaultValueTypeMapping) + _require_enough_parents(self, DefaultValueIndexNameMapping) class ExpandedParameterDefaultValueMapping(ImportMapping): @@ -759,69 +840,90 @@ class ExpandedParameterDefaultValueMapping(ImportMapping): Whenever this mapping is a child of :class:`ParameterDefaultValueIndexMapping`, it maps individual values of indexed parameters. - - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping` as parent. """ MAP_TYPE = "ExpandedDefaultValue" def _import_row(self, source_data, state, mapped_data): - values = state.setdefault(ImportKey.PARAMETER_DEFAULT_VALUES, {}) - value = values[_default_value_key(state)] - val = source_data - data = value.setdefault("data", []) - if value["type"] == "array": - data.append(val) - return - indexes = state.pop(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES) - data.append(indexes + [val]) + record = state[ImportKey.PARAMETER_DEFAULT_VALUE_RECORD] + record.values.append(source_data) + try: + record.indexes.append(state.pop(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES)) + except KeyError: + pass + try: + index_names = state.pop(ImportKey.PARAMETER_DEFAULT_VALUE_INDEX_NAMES) + except KeyError: + pass + else: + if record.indexes: + n_indexes = len(record.indexes[-1]) + if n_indexes == len(index_names): + record.index_names = list(index_names.values()) + else: + name_list = [] + for i in range(n_indexes): + name_list.append(index_names.get(i)) + record.index_names = name_list + else: + # Arrays + record.index_names = [index_names[0]] def _skip_row(self, state): - state.pop(ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES, None) + try: + del state[ImportKey.PARAMETER_DEFAULT_VALUE_INDEXES] + except KeyError: + pass + def check_validity(self) -> None: + _require_parent(self, ParameterDefaultValueTypeMapping) -class ParameterValueMapping(ImportMapping): - """Maps scalar (non-indexed) parameter values. - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping`, an entity mapping and - an :class:`AlternativeMapping` as parents. - """ +class ParameterValueMapping(ImportMapping): + """Maps scalar (non-indexed) parameter values.""" MAP_TYPE = "ParameterValue" def _import_row(self, source_data, state, mapped_data): - value = source_data - if value == "": + if source_data == "": return - entity_class_name, entity_byname, parameter_name, alternative_name = _parameter_value_key(state, mapped_data) - parameter_value = [entity_class_name, entity_byname, parameter_name, value] - if alternative_name is not None: - parameter_value.append(alternative_name) - mapped_data.setdefault("parameter_values", []).append(parameter_value) + entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) + entity_name = state[ImportKey.ENTITY_NAME] + entity_byname = entity_name if isinstance(entity_name, tuple) else (entity_name,) + parameter_name = state[ImportKey.PARAMETER_NAME] + alternative_name = state.get(ImportKey.ALTERNATIVE_NAME) + mapped_data.setdefault("parameter_values", {})[ + entity_class_name, entity_byname, parameter_name, alternative_name + ] = source_data + + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) + _require_one_of_parents(self, (EntityMapping, ElementMapping)) class ParameterValueTypeMapping(IndexedValueMixin, ImportMapping): + """Maps indexed parameter values.""" + MAP_TYPE = "ParameterValueType" def _import_row(self, source_data, state, mapped_data): - if ImportKey.PARAMETER_NAME not in state: - # Don't catch errors here, this one's invisible - return - key = _parameter_value_key(state, mapped_data) - values = state.setdefault(ImportKey.PARAMETER_VALUES, {}) - if key in values: + entity_class_name = state.get(ImportKey.ENTITY_CLASS_NAME) + entity_name = state[ImportKey.ENTITY_NAME] + entity_byname = entity_name if isinstance(entity_name, tuple) else (entity_name,) + parameter_name = state[ImportKey.PARAMETER_NAME] + alternative_name = state.get(ImportKey.ALTERNATIVE_NAME) + key = entity_class_name, entity_byname, parameter_name, alternative_name + mapped_values = mapped_data.setdefault("parameter_values", {}) + if key in mapped_values: + state[ImportKey.PARAMETER_VALUE_RECORD] = mapped_values[key] return - entity_class_name, entity_byname, parameter_name, alternative_name = key - value_type = str(source_data) - value = values[key] = {"type": value_type} # See import_mapping.generator._parameter_value_from_dict() - if self.compress and value_type == "map": - value["compress"] = self.compress - if self.options and value_type == "time_series": - value["options"] = self.options - parameter_value = [entity_class_name, entity_byname, parameter_name, value] - if alternative_name is not None: - parameter_value.append(alternative_name) - mapped_data.setdefault("parameter_values", []).append(parameter_value) + record = self._make_value_record(source_data) + mapped_values[key] = record + state[ImportKey.PARAMETER_VALUE_RECORD] = record + + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) + _require_one_of_parents(self, (EntityMapping, ElementMapping)) class ParameterValueMetadataNameMapping(ImportMapping): @@ -835,11 +937,7 @@ def _import_row(self, source_data, state, mapped_data): class ParameterValueMetadataValueMapping(ImportMapping): - """Maps parameter value metadata values. - - Cannot be used as the topmost mapping; must have a :class:`ParameterValueMapping` or - a :class:`ParameterValueTypeMapping` and :class:`ParameterValueMetadataName` as parents. - """ + """Maps parameter value metadata values.""" MAP_TYPE = "ParameterValueMetadataValue" ignorable = True @@ -855,34 +953,37 @@ def _import_row(self, source_data, state, mapped_data): entity_class_name, entity_byname, parameter_name, metadata_name, metadata_value, alternative_name ] = None + def check_validity(self) -> None: + _require_parent(self, ParameterValueMetadataNameMapping) -class ParameterValueIndexMapping(ImportMapping): - """Maps parameter value indexes. - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping`, an entity mapping and - an :class:`ParameterValueTypeMapping` as parents. - """ +class ParameterValueIndexMapping(ImportMapping): + """Maps parameter value indexes.""" MAP_TYPE = "ParameterValueIndex" def _import_row(self, source_data, state, mapped_data): - _ = state[ImportKey.PARAMETER_NAME] - index = source_data - state.setdefault(ImportKey.PARAMETER_VALUE_INDEXES, []).append(index) + state.setdefault(ImportKey.PARAMETER_VALUE_INDEXES, []).append(source_data) + def check_validity(self) -> None: + _require_parent(self, ParameterValueTypeMapping) + _require_enough_parents(self, IndexNameMapping) -class IndexNameMapping(IndexNameMappingBase): - """Maps index names for indexed parameter values. - Cannot be used as the topmost mapping; must have an :class:`ParameterValueTypeMapping` as a parent. - """ +class IndexNameMapping(ImportMapping): + """Maps index names for indexed parameter values.""" MAP_TYPE = "IndexName" - _STATE_KEY = ImportKey.PARAMETER_VALUES - @staticmethod - def _value_key(state, mapped_data): - return _parameter_value_key(state, mapped_data) + def _import_row(self, source_data, state, mapped_data): + if ImportKey.PARAMETER_VALUE_INDEXES in state: + i = len(state[ImportKey.PARAMETER_VALUE_INDEXES]) + state.setdefault(ImportKey.PARAMETER_VALUE_INDEX_NAMES, {})[i] = str(source_data) + else: + state[ImportKey.PARAMETER_VALUE_INDEX_NAMES] = {0: str(source_data)} + + def check_validity(self) -> None: + _require_parent(self, ParameterValueTypeMapping) class ExpandedParameterValueMapping(ImportMapping): @@ -890,48 +991,66 @@ class ExpandedParameterValueMapping(ImportMapping): Whenever this mapping is a child of :class:`ParameterValueIndexMapping`, it maps individual values of indexed parameters. - - Cannot be used as the topmost mapping; must have a :class:`ParameterDefinitionMapping`, an entity mapping and - an :class:`ParameterValueTypeMapping` as parents. """ MAP_TYPE = "ExpandedValue" def _import_row(self, source_data, state, mapped_data): - values = state.setdefault(ImportKey.PARAMETER_VALUES, {}) - value = values[_parameter_value_key(state, mapped_data)] - data = value.setdefault("data", []) - if value["type"] == "array": - data.append(source_data) - return - indexes = state.pop(ImportKey.PARAMETER_VALUE_INDEXES) - data.append(indexes + [source_data]) + record = state[ImportKey.PARAMETER_VALUE_RECORD] + record.values.append(source_data) + try: + record.indexes.append(state.pop(ImportKey.PARAMETER_VALUE_INDEXES)) + except KeyError: + pass + try: + index_names = state.pop(ImportKey.PARAMETER_VALUE_INDEX_NAMES) + except KeyError: + pass + else: + if record.indexes: + n_indexes = len(record.indexes[-1]) + if n_indexes == len(index_names): + record.index_names = list(index_names.values()) + else: + name_list = [] + for i in range(n_indexes): + name_list.append(index_names.get(i)) + record.index_names = name_list + else: + # Arrays + record.index_names = [index_names[0]] def _skip_row(self, state): - state.pop(ImportKey.PARAMETER_VALUE_INDEXES, None) + try: + del state[ImportKey.PARAMETER_VALUE_INDEXES] + except KeyError: + pass + def check_validity(self) -> None: + _require_parent(self, ParameterValueTypeMapping) -class ParameterValueListMapping(ImportMapping): - """Maps parameter value list names. - Can be used as the topmost mapping; in case the mapping has a :class:`ParameterDefinitionMapping` as parent, - yields value list name for that parameter definition. - """ +class ParameterValueListMapping(ImportMapping): + """Maps parameter value list names.""" MAP_TYPE = "ParameterValueList" def _import_row(self, source_data, state, mapped_data): - if self.parent is not None: - # Trigger a KeyError in case there's no parameter definition, so check_validity() registers the issue - _ = state[ImportKey.PARAMETER_DEFINITION] - state[ImportKey.PARAMETER_VALUE_LIST_NAME] = str(source_data) + value_list_name = str(source_data) + if not value_list_name: + return + state[ImportKey.PARAMETER_VALUE_LIST_NAME] = value_list_name + if ImportKey.PARAMETER_NAME in state: + parameter_name = state[ImportKey.PARAMETER_NAME] + entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] + mapped_data["parameter_definitions"][entity_class_name, parameter_name].value_list_name = value_list_name + def check_validity(self) -> None: + _require_parent(self, ParameterDefinitionMapping) -class ParameterValueListValueMapping(ImportMapping): - """Maps parameter value list values. - Cannot be used as the topmost mapping; must have a :class:`ParameterValueListMapping` as parent. - """ +class ParameterValueListValueMapping(ImportMapping): + """Maps parameter value list values.""" MAP_TYPE = "ParameterValueListValue" @@ -942,12 +1061,12 @@ def _import_row(self, source_data, state, mapped_data): value_list_name = state[ImportKey.PARAMETER_VALUE_LIST_NAME] mapped_data.setdefault("parameter_value_lists", []).append([value_list_name, list_value]) + def check_validity(self) -> None: + _require_parent(self, ParameterValueListMapping) -class AlternativeMapping(ImportMapping): - """Maps alternatives. - Can be used as the topmost mapping. - """ +class AlternativeMapping(ImportMapping): + """Maps alternatives.""" MAP_TYPE = "Alternative" @@ -957,10 +1076,7 @@ def _import_row(self, source_data, state, mapped_data): class AlternativeDescriptionMapping(ImportMapping): - """Maps alternative descriptions. - - Cannot be used as the topmost mapping; must have a :class:`AlternativeMapping` as parent. - """ + """Maps alternative descriptions.""" MAP_TYPE = "AlternativeDescription" ignorable = True @@ -973,12 +1089,12 @@ def _import_row(self, source_data, state, mapped_data): alternative_data.discard(alternative) alternative_data.add((alternative, description)) + def check_validity(self) -> None: + _require_parent(self, AlternativeMapping) -class ScenarioMapping(ImportMapping): - """Maps scenarios. - Can be used as the topmost mapping. - """ +class ScenarioMapping(ImportMapping): + """Maps scenarios.""" MAP_TYPE = "Scenario" @@ -989,10 +1105,7 @@ def _import_row(self, source_data, state, mapped_data): class ScenarioAlternativeMapping(ImportMapping): - """Maps scenario alternatives. - - Cannot be used as the topmost mapping; must have a :class:`ScenarioMapping` as parent. - """ + """Maps scenario alternatives.""" MAP_TYPE = "ScenarioAlternative" @@ -1004,12 +1117,12 @@ def _import_row(self, source_data, state, mapped_data): scen_alt = state[ImportKey.SCENARIO_ALTERNATIVE] = [scenario, alternative] mapped_data.setdefault("scenario_alternatives", []).append(scen_alt) + def check_validity(self) -> None: + _require_parent(self, ScenarioMapping) -class ScenarioBeforeAlternativeMapping(ImportMapping): - """Maps scenario 'before' alternatives. - Cannot be used as the topmost mapping; must have a :class:`ScenarioAlternativeMapping` as parent. - """ +class ScenarioBeforeAlternativeMapping(ImportMapping): + """Maps scenario 'before' alternatives.""" MAP_TYPE = "ScenarioBeforeAlternative" @@ -1018,12 +1131,12 @@ def _import_row(self, source_data, state, mapped_data): alternative = str(source_data) scen_alt.append(alternative) + def check_validity(self) -> None: + _require_parent(self, ScenarioAlternativeMapping) -class ScenarioDescriptionMapping(ImportMapping): - """Maps scenario descriptions. - Cannot be used as the topmost mapping; must have a :class:`ScenarioMapping` as parent. - """ +class ScenarioDescriptionMapping(ImportMapping): + """Maps scenario descriptions.""" MAP_TYPE = "ScenarioDescription" ignorable: ClassVar[bool] = True @@ -1036,6 +1149,9 @@ def _import_row(self, source_data, state, mapped_data): scenario_data.discard((scenario,)) scenario_data.add((scenario, description)) + def check_validity(self) -> None: + _require_parent(self, ScenarioMapping) + def default_import_mapping(map_type: str) -> ImportMapping: """Creates default mappings for given map type. @@ -1181,6 +1297,7 @@ def from_dict(serialized): ElementMapping, EntityAlternativeActivityMapping, ParameterDefinitionMapping, + ParameterDefinitionDescriptionMapping, ParameterTypeMapping, ParameterDefaultValueMapping, ParameterDefaultValueTypeMapping, diff --git a/tests/import_mapping/test_generator.py b/tests/import_mapping/test_generator.py index a0f58268..b0f52ac8 100644 --- a/tests/import_mapping/test_generator.py +++ b/tests/import_mapping/test_generator.py @@ -14,7 +14,7 @@ import unittest from spinedb_api import Array, DateTime, Duration, Map from spinedb_api.import_mapping.generator import get_mapped_data -from spinedb_api.import_mapping.import_mapping import default_import_mapping +from spinedb_api.import_mapping.import_mapping import EntityClassMapping, default_import_mapping from spinedb_api.import_mapping.type_conversion import value_to_convert_spec from spinedb_api.mapping import to_dict, unflatten @@ -71,7 +71,7 @@ def test_returns_appropriate_error_if_last_row_is_empty(self): "alternatives": {"Base"}, "entity_classes": [["Object", []]], "parameter_values": [["Object", ("data",), "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], - "parameter_definitions": [("Object", "Parameter")], + "parameter_definitions": [["Object", "Parameter"]], "entities": [["Object", "data"]], }, ) @@ -103,7 +103,7 @@ def test_convert_functions_get_expanded_over_last_defined_column_in_pivoted_data "alternatives": {"Base"}, "entity_classes": [["Object", []]], "parameter_values": [["Object", ("data",), "Parameter", Map(["T1", "T2"], [5.0, 99.0]), "Base"]], - "parameter_definitions": [("Object", "Parameter")], + "parameter_definitions": [["Object", "Parameter"]], "entities": [["Object", "data"]], }, ) @@ -134,7 +134,7 @@ def test_read_start_row_skips_rows_in_pivoted_data(self): { "entity_classes": [["klass", []]], "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], - "parameter_definitions": [("klass", "Parameter_2")], + "parameter_definitions": [["klass", "Parameter_2"]], "entities": [["klass", "kloss"]], }, ) @@ -187,8 +187,7 @@ def test_map_without_values_is_ignored_and_not_interpreted_as_null(self): { "alternatives": {"base"}, "entity_classes": [["o", []]], - "parameter_definitions": [("o", "parameter_name")], - "parameter_values": [], + "parameter_definitions": [["o", "parameter_name"]], "entities": [["o", "o1"]], }, ) @@ -232,7 +231,7 @@ def test_import_object_works_with_multiple_relationship_object_imports(self): ["q", "q2"], ["o_to_q", ["o1", "q2"]], ], - "parameter_definitions": [("o_to_q", "param")], + "parameter_definitions": [["o_to_q", "param"]], "parameter_values": [ ["o_to_q", ("o1", "q1"), "param", Map(["t1", "t2"], [11, 22], index_name="time"), "base"], ["o_to_q", ("o2", "q2"), "param", Map(["t1", "t2"], [33, 44], index_name="time"), "base"], @@ -268,7 +267,7 @@ def test_default_convert_function_in_column_convert_functions(self): { "entity_classes": [["klass", []]], "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], - "parameter_definitions": [("klass", "Parameter_2")], + "parameter_definitions": [["klass", "Parameter_2"]], "entities": [["klass", "kloss"]], }, ) @@ -296,7 +295,7 @@ def test_identity_function_is_used_as_convert_function_when_no_convert_functions { "entity_classes": [["klass", []]], "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], ["2.3", "23.0"])]], - "parameter_definitions": [("klass", "Parameter_2")], + "parameter_definitions": [["klass", "Parameter_2"]], "entities": [["klass", "kloss"]], }, ) @@ -326,7 +325,7 @@ def test_last_convert_function_gets_used_as_default_convert_function_when_no_def { "entity_classes": [["klass", []]], "parameter_values": [["klass", ("kloss",), "Parameter_2", Map(["T1", "T2"], [2.3, 23.0])]], - "parameter_definitions": [("klass", "Parameter_2")], + "parameter_definitions": [["klass", "Parameter_2"]], "entities": [["klass", "kloss"]], }, ) @@ -362,7 +361,7 @@ def test_array_parameters_get_imported_correctly_when_objects_are_in_header(self ["class", ("object_1",), "param", Array([-1.1, 1.1]), "Base"], ["class", ("object_2",), "param", Array([2.3, -2.3]), "Base"], ], - "parameter_definitions": [("class", "param")], + "parameter_definitions": [["class", "param"]], "entities": [["class", "object_1"], ["class", "object_2"]], }, ) @@ -398,7 +397,7 @@ def test_arrays_get_imported_correctly_when_objects_are_in_header_and_alternativ ["Gadget", ("object_1",), "data", Array([-1.1, 1.1]), "Base"], ["Gadget", ("object_2",), "data", Array([2.3, -2.3]), "Base"], ], - "parameter_definitions": [("Gadget", "data")], + "parameter_definitions": [["Gadget", "data"]], "entities": [["Gadget", "object_1"], ["Gadget", "object_2"]], }, ) @@ -435,7 +434,7 @@ def test_header_position_is_ignored_in_last_mapping_if_other_mappings_are_in_hea ["Data", ("d2",), "parameter1", -1.1, "Base"], ["Data", ("d2",), "parameter2", 2.3, "Base"], ], - "parameter_definitions": [("Data", "parameter1"), ("Data", "parameter2")], + "parameter_definitions": [["Data", "parameter1"], ["Data", "parameter2"]], "entities": [["Data", "d1"], ["Data", "d2"]], }, ) @@ -504,7 +503,7 @@ def test_importing_multidimensional_class_when_there_is_an_extra_column(self): ["node", "Ring world"], ], "entity_classes": [["unit__node__node", ["unit", "node", "node"]], ["unit", []], ["node", []]], - "parameter_definitions": [("unit__node__node", "flow")], + "parameter_definitions": [["unit__node__node", "flow"]], "parameter_values": [ ["unit__node__node", ("Dyson sphere", "Gamma Ceti", "Ring world"), "flow", 23.3, "Base"] ], @@ -536,7 +535,7 @@ def test_importing_empty_rows_does_unnecessarily_not_repeat_mapped_data(self): { "entities": [["Generator", "MyHydroGenerator"]], "entity_classes": [["Generator", []]], - "parameter_definitions": [("Generator", "Type")], + "parameter_definitions": [["Generator", "Type"]], "parameter_values": [["Generator", ("MyHydroGenerator",), "Type", "Hydro"]], }, ) @@ -601,7 +600,7 @@ def test_pivoted_mapping_has_position_outside_source_bounds(self): ["connection", []], ["node", []], ], - "parameter_definitions": [("connection__node__node", "flow_t")], + "parameter_definitions": [["connection__node__node", "flow_t"]], "parameter_values": [ [ "connection__node__node", @@ -682,7 +681,7 @@ def test_import_datetime_values(self): ["Object", "o1"], ["Object", "o2"], ], - "parameter_definitions": [("Object", "t")], + "parameter_definitions": [["Object", "t"]], "parameter_values": [ ["Object", ("o1",), "t", DateTime("2024-06-24T09:00:00"), "Base"], ["Object", ("o2",), "t", DateTime("2024-06-24T00:00:00"), "Base"], @@ -717,7 +716,7 @@ def test_import_durations(self): ["Object", "o1"], ["Object", "o2"], ], - "parameter_definitions": [("Object", "t")], + "parameter_definitions": [["Object", "t"]], "parameter_values": [ ["Object", ("o1",), "t", Duration("23D"), "Base"], ["Object", ("o2",), "t", Duration("19D"), "Base"], @@ -946,7 +945,7 @@ def test_import_parameter_types(self): mapped_data, { "entity_classes": [["Widget", []], ["Gadget", []], ["Object", []]], - "parameter_definitions": [("Widget", "x"), ("Gadget", "p"), ("Gadget", "q"), ("Object", "w")], + "parameter_definitions": [["Widget", "x"], ["Gadget", "p"], ["Gadget", "q"], ["Object", "w"]], "parameter_types": [ ("Widget", "x", "float"), ("Widget", "x", "bool"), @@ -1076,7 +1075,7 @@ def test_missing_entity_alternative_does_not_prevent_importing_of_values(self): "entities": [["unit", "Wind_plant"]], "entity_alternatives": [("unit", ("Wind_plant",), "Succeed", True)], "entity_classes": [["unit", []]], - "parameter_definitions": [("unit", "existing")], + "parameter_definitions": [["unit", "existing"]], "parameter_values": [ ["unit", ("Wind_plant",), "existing", 150.0, "Fail"], ["unit", ("Wind_plant",), "existing", 200.0, "Succeed"], @@ -1172,7 +1171,7 @@ def test_import_parameter_value_metadata(self): "alternatives": {"Base"}, "entities": [["cat", "Garfield"], ["cat", "Tom"]], "entity_classes": [["cat", []]], - "parameter_definitions": [("cat", "weight")], + "parameter_definitions": [["cat", "weight"]], "parameter_value_metadata": [ ("cat", ("Garfield",), "weight", "Tools", "Harrison-Stetson 1.0", "Base"), ("cat", ("Garfield",), "weight", "Licences", "Public domain", "Base"), diff --git a/tests/import_mapping/test_import_mapping.py b/tests/import_mapping/test_import_mapping.py index a14df131..d1184281 100644 --- a/tests/import_mapping/test_import_mapping.py +++ b/tests/import_mapping/test_import_mapping.py @@ -26,6 +26,7 @@ IndexNameMapping, ParameterDefaultValueIndexMapping, ParameterDefaultValueTypeMapping, + ParameterDefinitionDescriptionMapping, ParameterDefinitionMapping, ParameterValueIndexMapping, ParameterValueMapping, @@ -62,7 +63,7 @@ def test_convert_functions_float(self): expected = { "entity_classes": [["a", []]], "entities": [["a", "obj"]], - "parameter_definitions": [("a", "param", 1.2)], + "parameter_definitions": [["a", "param", 1.2]], } self.assertEqual(mapped_data, expected) @@ -81,7 +82,7 @@ def test_convert_functions_str(self): expected = { "entity_classes": [["a", []]], "entities": [["a", "obj"]], - "parameter_definitions": [("a", "param", "1111.2222")], + "parameter_definitions": [["a", "param", "1111.2222"]], } self.assertEqual(mapped_data, expected) @@ -100,7 +101,7 @@ def test_convert_functions_bool(self): expected = { "entity_classes": [["a", []]], "entities": [["a", "obj"]], - "parameter_definitions": [("a", "param", False)], + "parameter_definitions": [["a", "param", False]], } self.assertEqual(mapped_data, expected) @@ -737,8 +738,11 @@ def test_invalid_single_value_mapping_missing_parameter_definition(self): self.assertTrue(issues) def test_valid_array_mapping(self): - value_mapping = parameter_value_mapping_from_dict({"value_type": "array"}) - issues = check_validity(value_mapping) + root_mapping = default_import_mapping("EntityClass") + root_mapping.position = 0 + definition_mapping = root_mapping.tail_mapping().child = parameter_mapping_from_dict({"value_type": "array"}) + definition_mapping.position = 3 + issues = check_validity(root_mapping) self.assertFalse(issues) def test_invalid_array_mapping_missing_parameter_definition(self): @@ -748,8 +752,13 @@ def test_invalid_array_mapping_missing_parameter_definition(self): self.assertTrue(issues) def test_valid_time_series_mapping(self): - value_mapping = parameter_value_mapping_from_dict({"value_type": "time_series"}) - issues = check_validity(value_mapping) + root_mapping = default_import_mapping("EntityClass") + root_mapping.position = 0 + definition_mapping = root_mapping.tail_mapping().child = parameter_mapping_from_dict( + {"value_type": "time_series"} + ) + definition_mapping.position = 3 + issues = check_validity(root_mapping) self.assertFalse(issues) def test_invalid_time_series_mapping_missing_parameter_definition(self): @@ -815,7 +824,7 @@ def test_read_flat_file(self): expected = { "entity_classes": [["oc1", []], ["oc2", []]], "entities": [["oc1", "obj1"], ["oc2", "obj2"]], - "parameter_definitions": [("oc1", "parameter_name1"), ("oc2", "parameter_name2")], + "parameter_definitions": [["oc1", "parameter_name1"], ["oc2", "parameter_name2"]], "parameter_values": [["oc1", ("obj1",), "parameter_name1", 1], ["oc2", ("obj2",), "parameter_name2", 2]], } @@ -842,7 +851,7 @@ def test_read_flat_file_array(self): expected = { "entity_classes": [["oc1", []]], "entities": [["oc1", "obj1"]], - "parameter_definitions": [("oc1", "parameter_name1")], + "parameter_definitions": [["oc1", "parameter_name1"]], "parameter_values": [["oc1", ("obj1",), "parameter_name1", Array([1, 2])]], } @@ -869,7 +878,7 @@ def test_read_flat_file_array_with_ed(self): expected = { "entity_classes": [["oc1", []]], "entities": [["oc1", "obj1"]], - "parameter_definitions": [("oc1", "parameter_name1")], + "parameter_definitions": [["oc1", "parameter_name1"]], "parameter_values": [["oc1", ("obj1",), "parameter_name1", Array([1, 2])]], } @@ -960,7 +969,7 @@ def test_read_pivoted_parameters_from_header(self): expected = { "entity_classes": [["object", []]], "entities": [["object", "obj1"], ["object", "obj2"]], - "parameter_definitions": [("object", "parameter_name1"), ("object", "parameter_name2")], + "parameter_definitions": [["object", "parameter_name1"], ["object", "parameter_name2"]], "parameter_values": [ ["object", ("obj1",), "parameter_name1", 0], ["object", ("obj1",), "parameter_name2", 1], @@ -1006,7 +1015,7 @@ def test_read_pivoted_parameters_from_data(self): expected = { "entity_classes": [["object", []]], "entities": [["object", "obj1"], ["object", "obj2"]], - "parameter_definitions": [("object", "parameter_name1"), ("object", "parameter_name2")], + "parameter_definitions": [["object", "parameter_name1"], ["object", "parameter_name2"]], "parameter_values": [ ["object", ("obj1",), "parameter_name1", 0], ["object", ("obj1",), "parameter_name2", 1], @@ -1040,7 +1049,7 @@ def test_pivoted_value_has_actual_position(self): expected = { "entity_classes": [["timeline", []]], "entities": [["timeline", "obj1"], ["timeline", "obj2"]], - "parameter_definitions": [("timeline", "value")], + "parameter_definitions": [["timeline", "value"]], "alternatives": {"Base"}, "parameter_values": [ ["timeline", ("obj1",), "value", Map(["T1", "T2"], [11.0, 12.0], index_name="timestep"), "Base"], @@ -1070,7 +1079,7 @@ def test_import_objects_from_pivoted_data_when_they_lack_parameter_values(self): expected = { "entity_classes": [["node", []]], "entities": [["node", "obj1"], ["node", "obj2"]], - "parameter_definitions": [("node", "is_skilled"), ("node", "has_powers")], + "parameter_definitions": [["node", "is_skilled"], ["node", "has_powers"]], "alternatives": {"Base"}, "parameter_values": [ ["node", ("obj1",), "is_skilled", "yes", "Base"], @@ -1101,7 +1110,7 @@ def test_import_objects_from_pivoted_data_when_they_lack_map_type_parameter_valu expected = { "entity_classes": [["node", []]], "entities": [["node", "obj1"]], - "parameter_definitions": [("node", "is_skilled"), ("node", "has_powers")], + "parameter_definitions": [["node", "is_skilled"], ["node", "has_powers"]], "alternatives": {"Base"}, "parameter_values": [ [ @@ -1136,7 +1145,7 @@ def test_read_flat_file_with_extra_value_dimensions(self): expected = { "entity_classes": [["object", []]], "entities": [["object", "obj1"]], - "parameter_definitions": [("object", "parameter_name1")], + "parameter_definitions": [["object", "parameter_name1"]], "parameter_values": [ [ "object", @@ -1173,7 +1182,7 @@ def test_read_flat_file_with_parameter_definition(self): expected = { "entity_classes": [["object", []]], "entities": [["object", "obj1"]], - "parameter_definitions": [("object", "parameter_name1")], + "parameter_definitions": [["object", "parameter_name1"]], } data = iter(input_data) @@ -1245,7 +1254,7 @@ def test_read_relationships_with_parameters(self): expected = { "entity_classes": [["unit__node", ["unit", "node"]]], "entities": [["unit__node", ["u1", "n1"]], ["unit__node", ["u1", "n2"]]], - "parameter_definitions": [("unit__node", "rel_parameter")], + "parameter_definitions": [["unit__node", "rel_parameter"]], "parameter_values": [ ["unit__node", ("u1", "n1"), "rel_parameter", 0], ["unit__node", ("u1", "n2"), "rel_parameter", 1], @@ -1281,7 +1290,7 @@ def test_read_relationships_with_parameters2(self): ["nuts2__fueltype", ["DE11", "Bioenergy"]], ["nuts2", "DE11"], ], - "parameter_definitions": [("nuts2__fueltype", "capacity")], + "parameter_definitions": [["nuts2__fueltype", "capacity"]], "parameter_values": [ ["nuts2__fueltype", ("BE23", "Bioenergy"), "capacity", 268.0], ["nuts2__fueltype", ("DE11", "Bioenergy"), "capacity", 14.0], @@ -1319,7 +1328,7 @@ def test_read_parameter_header_with_only_one_parameter(self): expected = { "entity_classes": [["object", []]], "entities": [["object", "obj1"], ["object", "obj2"]], - "parameter_definitions": [("object", "parameter_name1")], + "parameter_definitions": [["object", "parameter_name1"]], "parameter_values": [ ["object", ("obj1",), "parameter_name1", 0], ["object", ("obj2",), "parameter_name1", 2], @@ -1345,7 +1354,7 @@ def test_read_pivoted_parameters_from_data_with_skipped_column(self): expected = { "entity_classes": [["object", []]], "entities": [["object", "obj1"], ["object", "obj2"]], - "parameter_definitions": [("object", "parameter_name1")], + "parameter_definitions": [["object", "parameter_name1"]], "parameter_values": [ ["object", ("obj1",), "parameter_name1", 0], ["object", ("obj2",), "parameter_name1", 2], @@ -1406,7 +1415,7 @@ def test_read_relationships_parameter_values_with_extra_dimensions(self): input_data = [["", "a", "b"], ["", "c", "d"], ["", "e", "f"], ["a", 2, 3], ["b", 4, 5]] expected = { "entity_classes": [["unit__node", ["unit", "node"]]], - "parameter_definitions": [("unit__node", "e"), ("unit__node", "f")], + "parameter_definitions": [["unit__node", "e"], ["unit__node", "f"]], "entities": [["unit__node", ["a", "c"]], ["unit__node", ["b", "d"]]], "parameter_values": [ ["unit__node", ("a", "c"), "e", Map(["a", "b"], [2, 4])], @@ -1444,7 +1453,7 @@ def test_read_data_with_read_start_row(self): expected = { "entity_classes": [["oc1", []], ["oc2", []]], "entities": [["oc1", "obj1"], ["oc2", "obj2"]], - "parameter_definitions": [("oc1", "parameter_name1"), ("oc2", "parameter_name2")], + "parameter_definitions": [["oc1", "parameter_name1"], ["oc2", "parameter_name2"]], "parameter_values": [["oc1", ("obj1",), "parameter_name1", 1], ["oc2", ("obj2",), "parameter_name2", 2]], } @@ -1473,7 +1482,7 @@ def test_read_data_with_two_mappings_with_different_read_start_row(self): expected = { "entity_classes": [["oc1", []], ["oc2", []]], "entities": [["oc1", "oc1_obj1"], ["oc1", "oc1_obj2"], ["oc2", "oc2_obj2"]], - "parameter_definitions": [("oc1", "parameter_class1"), ("oc2", "parameter_class2")], + "parameter_definitions": [["oc1", "parameter_class1"], ["oc2", "parameter_class2"]], "parameter_values": [ ["oc1", ("oc1_obj1",), "parameter_class1", 1], ["oc1", ("oc1_obj2",), "parameter_class1", 2], @@ -1542,7 +1551,7 @@ def test_read_flat_map_from_columns(self): "entity_classes": [["object_class", []]], "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1569,7 +1578,7 @@ def test_read_nested_map_from_columns(self): "entity_classes": [["object_class", []]], "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1613,7 +1622,7 @@ def test_read_uneven_nested_map_from_columns(self): "entity_classes": [["object_class", []]], "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1655,7 +1664,7 @@ def test_read_nested_map_with_compression(self): "entity_classes": [["object_class", []]], "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1867,9 +1876,9 @@ def test_read_parameter_definition_with_default_values_and_value_lists(self): expected = { "entity_classes": [["class_A", []], ["class_B", []]], "parameter_definitions": [ - ("class_A", "param1", 23.0, "listA"), - ("class_A", "param2", 42.0, "listB"), - ("class_B", "param3", 5.0, "listA"), + ["class_A", "param1", 23.0, "listA"], + ["class_A", "param2", 42.0, "listB"], + ["class_B", "param3", 5.0, "listA"], ], } self.assertFalse(errors) @@ -1891,7 +1900,7 @@ def test_map_as_default_parameter_value(self): expected_map = Map(["key1", "key2", "key3"], [-2.3, 5.5, 3.2]) expected = { "entity_classes": [["object_class", []]], - "parameter_definitions": [("object_class", "parameter", expected_map)], + "parameter_definitions": [["object_class", "parameter", expected_map]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1913,7 +1922,7 @@ def test_read_parameter_definition_with_nested_map_as_default_value(self): expected_map = Map(["key11", "key21"], [Map(["key12"], [-2]), Map(["key22"], [-1])]) expected = { "entity_classes": [["object_class", []]], - "parameter_definitions": [("object_class", "parameter", expected_map)], + "parameter_definitions": [["object_class", "parameter", expected_map]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1945,7 +1954,7 @@ def test_read_map_index_names_from_columns(self): "entity_classes": [["object_class", []]], "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -1977,7 +1986,7 @@ def test_missing_map_index_name(self): "entity_classes": [["object_class", []]], "entities": [["object_class", "object"]], "parameter_values": [["object_class", ("object",), "parameter", expected_map]], - "parameter_definitions": [("object_class", "parameter")], + "parameter_definitions": [["object_class", "parameter"]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -2006,7 +2015,7 @@ def test_read_default_value_index_names_from_columns(self): ) expected = { "entity_classes": [["object_class", []]], - "parameter_definitions": [("object_class", "parameter", expected_map)], + "parameter_definitions": [["object_class", "parameter", expected_map]], } self.assertFalse(errors) self.assertEqual(out, expected) @@ -2055,7 +2064,7 @@ def test_arrays_get_imported_to_correct_alternatives(self): expected = { "entity_classes": [["class", []]], "entities": [["class", "y"]], - "parameter_definitions": [("class", "parameter")], + "parameter_definitions": [["class", "parameter"]], "alternatives": {"Base", "alternative"}, "parameter_values": [ ["class", ("y",), "parameter", Array(["p1"]), "Base"], @@ -2130,3 +2139,39 @@ def test_mappings_are_hidden(self): root = default_import_mapping(map_type) flattened = root.flatten() self.assertTrue(all(m.position == Position.hidden for m in flattened)) + + +class TestParameterDefinitionDescriptionMapping: + def test_imports_correctly(self): + data_source = iter( + [ + ["Gadget", "weight", "Weight of a non-widget."], + ] + ) + flattened = [EntityClassMapping(0), ParameterDefinitionMapping(1), ParameterDefinitionDescriptionMapping(2)] + root_mapping = unflatten(flattened) + mapped_data, errors = get_mapped_data(data_source, [root_mapping]) + assert errors == [] + assert mapped_data == { + "entity_classes": [ + ["Gadget", []], + ], + "parameter_definitions": [["Gadget", "weight", None, None, "Weight of a non-widget."]], + } + + def test_empty_description_is_skipped(self): + data_source = iter( + [ + ["Gadget", "weight", ""], + ] + ) + flattened = [EntityClassMapping(0), ParameterDefinitionMapping(1), ParameterDefinitionDescriptionMapping(2)] + root_mapping = unflatten(flattened) + mapped_data, errors = get_mapped_data(data_source, [root_mapping]) + assert errors == [] + assert mapped_data == { + "entity_classes": [ + ["Gadget", []], + ], + "parameter_definitions": [["Gadget", "weight"]], + } From 9689d3b1a0428577cddf660ce9cd1718f7c5ef30 Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Wed, 4 Mar 2026 10:35:32 +0100 Subject: [PATCH 6/9] Fix issues with import mappings - ParameterValueListMapping cannot unconditionally require ParameterDefinitionMapping as parent. A smarter validation mechanism has been implemented to fix this. - We should let KeyErrors in ImportMapping.import_row() to propagate. They are now always logic errors as we do not use KeyErrors for mapping validation anymore. - Fixed a bug in EntityMapping where ND entities from the first entity class encountered on a table were imported but nothing else. Re spine-tools/Spine-Toolbox#1892 --- spinedb_api/exception.py | 5 ++ spinedb_api/import_mapping/import_mapping.py | 53 ++++++++++++-------- tests/import_mapping/test_generator.py | 38 ++++++++++++++ tests/import_mapping/test_import_mapping.py | 12 +++-- 4 files changed, 82 insertions(+), 26 deletions(-) diff --git a/spinedb_api/exception.py b/spinedb_api/exception.py index ec04c09c..9855de9c 100644 --- a/spinedb_api/exception.py +++ b/spinedb_api/exception.py @@ -77,6 +77,11 @@ def __init__(self, msg, rank=None, key=None): self.rank = rank self.key = key + def __eq__(self, other): + if not isinstance(other, InvalidMappingComponent): + return NotImplemented + return self.msg == other.msg and self.rank == other.rank and self.key == other.key + class ReaderError(SpineDBAPIError): """Failure in import reader.""" diff --git a/spinedb_api/import_mapping/import_mapping.py b/spinedb_api/import_mapping/import_mapping.py index 37c31935..bd58f3e6 100644 --- a/spinedb_api/import_mapping/import_mapping.py +++ b/spinedb_api/import_mapping/import_mapping.py @@ -86,9 +86,25 @@ def check_validity(root_mapping: ImportMapping) -> list[InvalidMappingComponent] mapping.check_validity() except InvalidMappingComponent as error: errors.append(error) + errors += _check_dependent_pairs(root_mapping) return errors +def _check_dependent_pairs(root_mapping: ImportMapping) -> list[InvalidMappingComponent]: + flattened = root_mapping.flatten() + try: + definition_mapping = next(m for m in flattened if isinstance(m, ParameterDefinitionMapping)) + value_list_mapping = next(m for m in flattened if isinstance(m, ParameterValueListMapping)) + except StopIteration: + return [] + if (value_list_mapping.position is not Position.hidden or definition_mapping.value is not None) and ( + definition_mapping.position == Position.hidden and definition_mapping.value is None + ): + value_list_rank = next(n for n, m in enumerate(flattened) if isinstance(m, ParameterValueListMapping)) + return [InvalidMappingComponent("value list requires a parameter name", value_list_rank)] + return [] + + class ImportMapping(Mapping): """Base class for import mappings.""" @@ -280,28 +296,20 @@ def filter_accepts_row(self, source_row): self.child is None or self.child.filter_accepts_row(source_row) ) - def import_row(self, source_row, state, mapped_data, errors=None): + def import_row(self, source_row, state, mapped_data): if self.has_filter() and not self.filter_accepts_row(source_row): return - if errors is None: - errors = [] if not (self.position == Position.hidden and self.value is None): source_data = self._data(source_row) if source_data is None: if not self.ignorable or self.child is None: self._skip_row(state) return - self.child.import_row(source_row, state, mapped_data, errors=errors) + self.child.import_row(source_row, state, mapped_data) return - try: - self._import_row(source_data, state, mapped_data) - except KeyError as err: - for key in err.args: - msg = f"Required key '{key}' is invalid" - error = InvalidMappingComponent(msg, self.rank, key) - errors.append(error) + self._import_row(source_data, state, mapped_data) if self.child is not None: - self.child.import_row(source_row, state, mapped_data, errors=errors) + self.child.import_row(source_row, state, mapped_data) def _data(self, source_row): # pylint: disable=arguments-renamed if source_row is None: @@ -525,7 +533,7 @@ def _import_row(self, source_data, state, mapped_data): mapped_data["entities"][entity_class_name, entity_name].description = description def check_validity(self) -> None: - _require_parent(self, EntityMapping) + _require_one_of_parents(self, (EntityMapping, ElementMapping)) class EntityMetadataNameMapping(ImportMapping): @@ -630,16 +638,20 @@ def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] if ImportKey.ENTITY_NAME in state: entity_name = state[ImportKey.ENTITY_NAME] - record = mapped_data["entities"][entity_class_name, entity_name] - if all(name == existing_name for name, existing_name in zip(element_names, record.elements)): - return - del state[ImportKey.ENTITY_NAME] + try: + record = mapped_data["entities"][entity_class_name, entity_name] + except KeyError: + pass + else: + if all(name == existing_name for name, existing_name in zip(element_names, record.elements)): + return + del state[ImportKey.ENTITY_NAME] record = EntityRecord(element_names) byname = tuple(record.elements) - mapped_data.setdefault("entities", {})[entity_class_name, byname] = record + mapped_entities = mapped_data.setdefault("entities", {}) + mapped_entities[entity_class_name, byname] = record state[ImportKey.ENTITY_NAME] = byname if self.import_entities: - mapped_entities = mapped_data.setdefault("entities", {}) mapped_classes = mapped_data["entity_classes"] class_record = mapped_classes[entity_class_name] for element_name, dimension_name in zip(element_names, class_record.dimensions): @@ -1045,9 +1057,6 @@ def _import_row(self, source_data, state, mapped_data): entity_class_name = state[ImportKey.ENTITY_CLASS_NAME] mapped_data["parameter_definitions"][entity_class_name, parameter_name].value_list_name = value_list_name - def check_validity(self) -> None: - _require_parent(self, ParameterDefinitionMapping) - class ParameterValueListValueMapping(ImportMapping): """Maps parameter value list values.""" diff --git a/tests/import_mapping/test_generator.py b/tests/import_mapping/test_generator.py index b0f52ac8..09c24bd5 100644 --- a/tests/import_mapping/test_generator.py +++ b/tests/import_mapping/test_generator.py @@ -1343,3 +1343,41 @@ def test_import_multidimensional_entities_with_descriptions(self): ], }, ) + + def test_import_different_relationships_in_same_table(self): + data_source = iter( + [ + ["Widget__Gadget", "Widget", "Gadget", "tableview", "watch", "A cozy relationship 1."], + ["Widget__Gadget", "Widget", "Gadget", "checkbox", "watch", "A cozy relationship 2."], + ["Widget__Gadget", "Widget", "Gadget", "checkbox", "clock", "A cozy relationship 3."], + ["Gadget__Widget", "Gadget", "Widget", "watch", "checkbox", "A cozy relationship 4."], + ["Gadget__Widget", "Gadget", "Widget", "clock", "tableview", "A cozy relationship 5."], + ] + ) + mappings = [ + [ + {"map_type": "EntityClass", "position": 0}, + {"map_type": "Dimension", "position": 1}, + {"map_type": "Dimension", "position": 2}, + {"map_type": "EntityClassDescription", "position": "hidden"}, + {"map_type": "Entity", "position": "hidden"}, + {"map_type": "Element", "position": 3}, + {"map_type": "Element", "position": 4}, + {"map_type": "EntityDescription", "position": 5}, + ] + ] + mapped_data, errors = get_mapped_data(data_source, mappings) + self.assertEqual(errors, []) + self.assertEqual( + mapped_data, + { + "entity_classes": [["Widget__Gadget", ["Widget", "Gadget"]], ["Gadget__Widget", ["Gadget", "Widget"]]], + "entities": [ + ["Widget__Gadget", ["tableview", "watch"], "A cozy relationship 1."], + ["Widget__Gadget", ["checkbox", "watch"], "A cozy relationship 2."], + ["Widget__Gadget", ["checkbox", "clock"], "A cozy relationship 3."], + ["Gadget__Widget", ["watch", "checkbox"], "A cozy relationship 4."], + ["Gadget__Widget", ["clock", "tableview"], "A cozy relationship 5."], + ], + }, + ) diff --git a/tests/import_mapping/test_import_mapping.py b/tests/import_mapping/test_import_mapping.py index d1184281..26c61675 100644 --- a/tests/import_mapping/test_import_mapping.py +++ b/tests/import_mapping/test_import_mapping.py @@ -13,7 +13,7 @@ """Unit tests for import Mappings.""" import unittest from unittest.mock import Mock -from spinedb_api.exception import InvalidMapping +from spinedb_api.exception import InvalidMapping, InvalidMappingComponent from spinedb_api.import_mapping.generator import get_mapped_data from spinedb_api.import_mapping.import_mapping import ( AlternativeMapping, @@ -550,14 +550,16 @@ def test_valid_object_default_value_mapping_not_missing_parameter_definition(sel issues = check_validity(cls_mapping) self.assertFalse(issues) - def test_invalid_object_value_list_mapping_missing_parameter_definition(self): + def test_value_list_mapping_missing_parameter_definition_is_ok(self): cls_mapping = import_mapping_from_dict({"map_type": "ObjectClass"}) cls_mapping.flatten()[-1].child = parameter_mapping_from_dict({"map_type": "ParameterDefinition"}) value_list_mapping = cls_mapping.flatten()[-2] cls_mapping.position = 0 value_list_mapping.position = 1 issues = check_validity(cls_mapping) - self.assertTrue(issues) + self.assertEqual( + issues, [InvalidMappingComponent("value list requires a parameter name", value_list_mapping.rank)] + ) def test_valid_object_value_list_mapping_not_missing_parameter_definition(self): cls_mapping = import_mapping_from_dict({"map_type": "ObjectClass"}) @@ -657,7 +659,9 @@ def test_invalid_relationship_value_list_mapping_missing_parameter_definition(se cls_mapping.position = 0 value_list_mapping.position = 1 issues = check_validity(cls_mapping) - self.assertTrue(issues) + self.assertEqual( + issues, [InvalidMappingComponent("value list requires a parameter name", value_list_mapping.rank)] + ) def test_valid_relationship_value_list_mapping_not_missing_parameter_definition(self): cls_mapping = import_mapping_from_dict({"map_type": "RelationshipClass"}) From 9439ec8b8876206989a4c4417b1c36948727c5ea Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Wed, 4 Mar 2026 13:16:33 +0100 Subject: [PATCH 7/9] Add parameter definition descriptions to parameter import mappings Re spine-tools/Spine-Toolbox#1892 --- spinedb_api/import_mapping/import_mapping_compat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spinedb_api/import_mapping/import_mapping_compat.py b/spinedb_api/import_mapping/import_mapping_compat.py index d4350b6d..a629e7e1 100644 --- a/spinedb_api/import_mapping/import_mapping_compat.py +++ b/spinedb_api/import_mapping/import_mapping_compat.py @@ -26,6 +26,7 @@ ParameterDefaultValueIndexMapping, ParameterDefaultValueMapping, ParameterDefaultValueTypeMapping, + ParameterDefinitionDescriptionMapping, ParameterDefinitionMapping, ParameterValueIndexMapping, ParameterValueListMapping, @@ -191,7 +192,8 @@ def parameter_mapping_from_dict(map_dict): if map_type == "ParameterDefinition": default_value_dict = map_dict.get("default_value") value_list_name = map_dict.get("parameter_value_list_name") - param_def_mapping.child = value_list_mapping = ParameterValueListMapping(*_pos_and_val(value_list_name)) + description = param_def_mapping.child = ParameterDefinitionDescriptionMapping(Position.hidden) + description.child = value_list_mapping = ParameterValueListMapping(*_pos_and_val(value_list_name)) value_list_mapping.child = parameter_default_value_mapping_from_dict(default_value_dict) return param_def_mapping alternative_name = map_dict.get("alternative_name") From 05d611bf934e34b23cc4f06a786fe03997408c08 Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Wed, 4 Mar 2026 13:45:04 +0100 Subject: [PATCH 8/9] Maybe fix a warning in unit tests --- tests/test_filtered_database_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_filtered_database_mapping.py b/tests/test_filtered_database_mapping.py index d0d96a88..c396e761 100644 --- a/tests/test_filtered_database_mapping.py +++ b/tests/test_filtered_database_mapping.py @@ -157,7 +157,7 @@ def test_rename_entity_to_something_that_has_been_filtered_out(tmp_path): bigglesworth = db_map.entity(name="Bigglesworth", entity_class_name="cat") bigglesworth.update(name="Tom") with pytest.raises( - SpineDBAPIError, match="^there's already a entity with \{'entity_class_name': 'cat', 'name': 'Tom'\}$" + SpineDBAPIError, match="^there's already a entity with \\{'entity_class_name': 'cat', 'name': 'Tom'\\}$" ): db_map.commit_session("Rename Bigglesworth.") assert bigglesworth.mapped_item.status == Status.to_update From e23dcc710c39374d1349e9022f01355e2720dc0f Mon Sep 17 00:00:00 2001 From: Antti Soininen Date: Thu, 5 Mar 2026 09:23:18 +0100 Subject: [PATCH 9/9] Require chardet >= 7 New chardet breaks our unit tests; however, it seems to be much more performant (it's a complete rewrite after all) and capable than the previous versions. So, let's switch to the latest and creates once and for all. --- pyproject.toml | 2 +- spinedb_api/spine_io/importers/csv_reader.py | 11 ++++++++-- tests/spine_io/importers/test_csv_reader.py | 2 +- .../importers/test_datapackage_reader.py | 20 +------------------ 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e16b6cf9..3535d54b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "openpyxl >=3.0.7, !=3.1.1", "GDX2py >=2.2.0", "ijson >=3.1.4", - "chardet >=4.0.0", + "chardet >=7", "PyMySQL[rsa] >=1.0.2", "psycopg2-binary", "pyarrow >= 19.0", diff --git a/spinedb_api/spine_io/importers/csv_reader.py b/spinedb_api/spine_io/importers/csv_reader.py index de2986b2..90d570e1 100644 --- a/spinedb_api/spine_io/importers/csv_reader.py +++ b/spinedb_api/spine_io/importers/csv_reader.py @@ -10,7 +10,7 @@ # this program. If not, see . ###################################################################################################################### -""" Contains CSVReader class and helper functions. """ +"""Contains CSVReader class and helper functions.""" import csv from itertools import islice @@ -44,6 +44,7 @@ class CSVReader(Reader): def __init__(self, settings): super().__init__(settings) self._filename = None + self._detector = chardet.UniversalDetector(max_bytes=1024) def connect_to_source(self, source, **extras): """saves filepath @@ -67,7 +68,13 @@ def get_tables_and_properties(self): options = {"skip": 0} # try to find options for file with open(self._filename, "rb") as input_file: - sniff_result = chardet.detect(input_file.read(1024)) + for line in input_file: + self._detector.feed(line) + if self._detector.done: + break + self._detector.close() + sniff_result = self._detector.result + self._detector.reset() sniffed_encoding = sniff_result["encoding"] if sniffed_encoding is not None: sniffed_encoding = sniffed_encoding.lower() diff --git a/tests/spine_io/importers/test_csv_reader.py b/tests/spine_io/importers/test_csv_reader.py index 79d22137..12b9628b 100644 --- a/tests/spine_io/importers/test_csv_reader.py +++ b/tests/spine_io/importers/test_csv_reader.py @@ -45,7 +45,7 @@ def test_get_tables_and_properties(self): self.assertEqual(len(tables), 1) self.assertTrue("data" in tables) options = tables["data"].options - self.assertEqual(options["encoding"], "ascii") + self.assertEqual(options["encoding"], "utf-8") self.assertEqual(options["delimiter"], ",") self.assertEqual(options["quotechar"], '"') self.assertEqual(options["skip"], 0) diff --git a/tests/spine_io/importers/test_datapackage_reader.py b/tests/spine_io/importers/test_datapackage_reader.py index 33ebd6eb..b23aefb9 100644 --- a/tests/spine_io/importers/test_datapackage_reader.py +++ b/tests/spine_io/importers/test_datapackage_reader.py @@ -13,6 +13,7 @@ import csv from pathlib import Path import pickle +import sys from tempfile import TemporaryDirectory import unittest from frictionless import Package, Resource @@ -53,25 +54,6 @@ def test_header_off_does_not_append_numbers_to_duplicate_cells(self): self.assertIsNone(header) self.assertEqual(list(data_iterator), data) - def test_wrong_datapackage_encoding_raises_reader_error(self): - broken_text = b"Slagn\xe4s" - # Fool the datapackage sniffing algorithm by hiding the broken line behind a large number of UTF-8 lines. - data = 1000 * [b"normal_text\n"] + [broken_text] - with TemporaryDirectory() as temp_dir: - csv_file_path = Path(temp_dir, "test_data.csv") - with open(csv_file_path, "wb") as csv_file: - for row in data: - csv_file.write(row) - package = Package(basepath=temp_dir) - package.add_resource(Resource(path=str(csv_file_path.relative_to(temp_dir)))) - package_path = Path(temp_dir, "datapackage.json") - package.to_json(package_path) - reader = DatapackageReader(None) - reader.connect_to_source(str(package_path)) - data_iterator, header = reader.get_data_iterator("test_data", {"has_header": False}) - self.assertIsNone(header) - self.assertRaises(ReaderError, list, data_iterator) - def test_get_table_cell(self): data = [["11", "12", "13"], ["21", "22", "23"]] with check_datapackage(data) as package_path: