Skip to content

API Reference

This API reference provides comprehensive documentation for all public classes and functions in the BAM Masterdata package. For more detailed examples and usage patterns, see the How-to Guides and Tutorial sections.

Warning

This documentation page is currently broken, as we shifted from MkDocs to Zensical for our documentation engine.

bam_masterdata.metadata.entities

BaseEntity

Bases: BaseModel

Base class used to define ObjectType and VocabularyType classes. It extends the BaseModel adding new methods that are useful for interfacing with openBIS.

Source code in bam_masterdata/metadata/entities.py
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
class BaseEntity(BaseModel):
    """
    Base class used to define `ObjectType` and `VocabularyType` classes. It extends the `BaseModel`
    adding new methods that are useful for interfacing with openBIS.
    """

    code: str | None = Field(
        default=None,
        description="""
        Code of the entity to assign as permanent identifier in openBIS.
        """,
    )

    def __init__(self, **kwargs):
        super().__init__()

        # We store the `_property_metadata` during instantiation of the class
        self._property_metadata = self.get_property_metadata()

        for key, value in kwargs.items():
            setattr(self, key, value)

    def __setattr__(self, key, value):
        # Allow initialization before metadata exists
        if not hasattr(self, "_property_metadata"):
            return object.__setattr__(self, key, value)

        if key == "_property_metadata":
            super().__setattr__(key, value)
            return

        if key in self._property_metadata:
            # TODO add CONTROLLEDVOCABULARY and OBJECT cases
            expected_type = self._property_metadata[key].data_type.pytype
            if expected_type and not isinstance(value, expected_type):
                raise TypeError(
                    f"Invalid type for '{key}': Expected {expected_type.__name__}, got {type(value).__name__}"
                )

        # TODO add check if someone tries to set up a definition instead of an assigned property

        object.__setattr__(self, key, value)

    def __repr__(self):
        # Filter for attributes that are `PropertyTypeAssignment` and set to a finite value
        class_prop_name = None
        fields = []
        for key, metadata in self._property_metadata.items():
            if isinstance(metadata, PropertyTypeAssignment):
                value = getattr(self, key, None)
                # Only include set attributes
                if value is not None and not isinstance(value, PropertyTypeAssignment):
                    if key == "name":
                        class_prop_name = value
                    fields.append(f"{key}={repr(value)}")

        # Format the output
        class_name = self.cls_name
        if class_prop_name:  # adding `name` if available
            class_name = f"{class_prop_name}:{class_name}"
        return f"{class_name}({', '.join(fields)})"

    # Overwriting the __str__ method to use the same representation as __repr__
    __str__ = __repr__

    @property
    def cls_name(self) -> str:
        """
        Returns the entity name of the class as a string to speed up checks. This is a property
        to be overwritten by each of the abstract entity types.
        """
        return self.__class__.__name__

    @property
    def _base_attrs(self) -> list:
        """
        List of base properties or terms assigned to an entity type. This are the direct properties or terms
        assigned when defining a new entity type.
        """
        cls_attrs = self.__class__.__dict__
        base_attrs = [
            attr_name
            for attr_name in cls_attrs
            if not (
                attr_name.startswith("_")
                or callable(cls_attrs[attr_name])
                or attr_name
                in ["defs", "model_config", "model_fields", "model_computed_fields"]
            )
        ]
        return [getattr(self, attr_name) for attr_name in base_attrs]

    @deprecated(
        "This method is deprecated and will be removed in a future major release. The creation of Object Types, "
        "Property Types, and Vocabularies is done now in the corresponding ObjectType and VocabularyType classes."
    )
    def _to_openbis(
        self,
        logger: "BoundLoggerLazyProxy",
        openbis: "Openbis",
        type: str,
        *,
        type_map: dict,
        get_type: Callable[..., Any],
        create_type: Callable[..., Any],
    ) -> None:
        """
        Simplified function to add or update the entity type in openBIS.
        """
        # Get all existing entities from openBIS
        openbis_entities = getattr(
            OpenbisEntities(url=openbis.url), f"get_{type}_dict"
        )()
        defs = getattr(self, "defs")

        is_vocab = isinstance(self, VocabularyType)

        # Check if the entity already exists
        if defs.code in openbis_entities:
            logger.info(f"Entity '{defs.code}' already exists in openBIS.")
            # Retrieve the existing entity
            entity = get_type(openbis, defs.code)
            # entity = openbis_entities[defs.code]

            # Get properties from self and openBIS
            self_properties = getattr(self, "terms" if is_vocab else "properties", [])
            obis_properties = (
                entity.get_terms().df.code
                if is_vocab
                else entity.get_property_assignments()
            )
            obis_property_codes = [
                prop.code if not is_vocab else prop for prop in obis_properties
            ]

            # Check for properties in self that are not in openBIS
            new_properties_added = False
            for prop in self_properties:
                if prop.code not in obis_property_codes:
                    logger.info(
                        f"Adding new '{'term' if is_vocab else 'property'}' {prop.code}' to entity '{defs.code}'."
                    )
                    new_properties_added = True

                    # Handle special case for OBJECT or SAMPLE data types
                    if not is_vocab and (
                        prop.data_type == "OBJECT" or prop.data_type == "SAMPLE"
                    ):
                        prop.data_type = "SAMPLE"

                    # Assign the term or property to the entity
                    if is_vocab:
                        term = openbis.new_term(
                            code=prop.code,
                            vocabularyCode=defs.code,
                            label=prop.label,
                            description=prop.description,
                        )
                        if prop.official:
                            term.official = prop.official
                        term.save()
                    else:
                        if prop.vocabulary_code:
                            entity.assign_property(
                                prop=prop.code,
                                section=prop.section,
                                mandatory=prop.mandatory,
                                showInEditView=prop.show_in_edit_views,
                                vocabulary=prop.vocabulary_code,
                                ordinal=prop.ordinal,
                            )
                        else:
                            entity.assign_property(
                                prop=prop.code,
                                section=prop.section,
                                mandatory=prop.mandatory,
                                showInEditView=prop.show_in_edit_views,
                                ordinal=prop.ordinal,
                            )

            if not new_properties_added:
                logger.info(
                    f"No new '{'terms' if is_vocab else 'properties'}' added to entity '{defs.code}'."
                )

            # Save the entity after adding new properties
            if not is_vocab:
                entity.save()
            return entity

        # If the entity is new, create it
        logger.info(f"Creating new entity '{defs.code}' in openBIS.")
        if not is_vocab:
            entity = create_type(openbis, defs)
            entity.save()

            # Assign properties to the new entity
            properties = getattr(self, "properties", [])
            for prop in properties:
                logger.info(f"Adding new property '{prop.code}' to '{defs.code}'.")
                # Handle special case for OBJECT or SAMPLE data types
                if prop.data_type == "OBJECT" or prop.data_type == "SAMPLE":
                    prop.data_type = "SAMPLE"

                # Assign the property to the entity
                entity.assign_property(
                    prop=prop.code,
                    section=prop.section,
                    mandatory=prop.mandatory,
                    showInEditView=prop.show_in_edit_views,
                    ordinal=prop.ordinal,
                )
        else:
            # Transform the list of VocabularyTerm objects into the desired format
            terms = [
                {
                    "code": term.code,
                    "label": term.label,
                    "description": term.description,
                }
                for term in getattr(self, "terms", [])
            ]
            term_codes = ", ".join([term.code for term in getattr(self, "terms", [])])
            logger.info(f"Adding new terms {term_codes} to '{defs.code}'.")
            entity = create_type(openbis, defs, terms)
            entity.save()

        # Save the entity after assigning properties
        if not is_vocab:
            entity.save()
        return entity

    def get_property_metadata(self) -> dict:
        """
        Dictionary containing the metadata of the properties assigned to the entity type.

        Returns:
            dict: A dictionary containing the keys of the `PropertyTypeAssignment` attribute names and the
            values of the definitions of `PropertyTypeAssignment`. Example:
            {
                "name": PropertyTypeAssignment(
                    code="$NAME",
                    data_type=VARCHAR,
                    mandatory=True,
                    property_label="Name"
                ),
                "age": PropertyTypeAssignment(
                    code="AGE",
                    data_type=INTEGER,
                    mandatory=False,
                    property_label="Age"
                ),
            }
        """
        cls_attrs = self.__class__.__dict__

        # Store property metadata at class level
        prop_meta_dict: dict = {}
        for base in type(self).__mro__:
            cls_attrs = getattr(base, "__dict__", {})
            for attr_name, attr_value in cls_attrs.items():
                if isinstance(attr_value, PropertyTypeAssignment):
                    prop_meta_dict[attr_name] = attr_value
        return prop_meta_dict

    def to_json(self, indent: int | None = None) -> str:
        """
        Returns the entity as a string in JSON format storing the value of the properties
        assigned to the entity.

        Args:
            indent (Optional[int], optional): The indent to print in JSON. Defaults to None.

        Returns:
            str: The JSON representation of the entity.
        """
        data: dict = {}
        for key in self._property_metadata.keys():
            try:
                value = getattr(self, key)
            except AttributeError:
                continue
            if isinstance(value, PropertyTypeAssignment):
                continue
            data[key] = value
        return json.dumps(data, indent=indent, sort_keys=True)

    def to_dict(self) -> dict:
        """
        Returns the entity as a dictionary storing the value of the properties assigned to the entity.

        Returns:
            dict: The dictionary representation of the entity.
        """
        dump_json = self.to_json()
        return json.loads(dump_json)

    def to_hdf5(self, hdf_file: h5py.File, group_name: str = "") -> h5py.File:
        """
        Serialize the entity to a HDF5 file under the group specified in the input.

        Args:
            hdf_file (h5py.File): The HDF5 file to store the entity.
            group_name (str, optional): The group name to serialize the data.
        """
        if not group_name:
            group_name = self.cls_name
        group = hdf_file.create_group(group_name)

        for key in self._property_metadata.keys():
            try:
                value = getattr(self, key)
                if not value:
                    continue
                if isinstance(value, str | int | float | bool | list | tuple):
                    group.create_dataset(key, data=value)
                else:
                    raise TypeError(
                        f"Unsupported type {type(value)} for key {key} for HDF5 serialization."
                    )
            except AttributeError:
                continue

    def model_to_dict(self) -> dict:
        """
        Returns the model as a dictionary storing the data `defs` and the property or vocabulary term
        assignments.

        Returns:
            dict: The dictionary representation of the model.
        """
        data = self.model_dump()

        attr_value = getattr(self, "defs")
        if isinstance(attr_value, BaseModel):
            data["defs"] = attr_value.model_dump()
        else:
            data["defs"] = attr_value
        return data

    def model_to_json(self, indent: int | None = None) -> str:
        """
        Returns the model as a string in JSON format storing the data `defs` and the property or
        vocabulary term assignments.

        Args:
            indent (Optional[int], optional): The indent to print in JSON. Defaults to None.

        Returns:
            str: The JSON representation of the model.
        """
        # * `model_dump_json()` from pydantic does not store the `defs` section of each entity.
        data = self.model_to_dict()
        return json.dumps(data, indent=indent)

    def _add_properties_rdf(
        self,
        namespace: "Namespace",
        graph: "Graph",
        prop: "PropertyTypeAssignment",
        logger: "BoundLoggerLazyProxy",
    ) -> "URIRef":
        """
        Add the properties assigned to the entity to the RDF graph extracting the information from
        OpenBIS for the `object_code` or `vocabulary_code`.

        Args:
            namespace (Namespace): The namespace to use for the RDF graph.
            graph (Graph): The RDF graph to which the properties are added.
            prop (PropertyTypeAssignment): The property assigned to the entity.
            logger (BoundLoggerLazyProxy): The logger to log messages.

        Returns:
            URIRef: The URI reference of the property added to the RDF graph.
        """
        prop_uri = namespace[prop.id]

        # Define the property as an OWL class inheriting from PropertyType
        graph.add((prop_uri, RDF.type, OWL.Thing))
        graph.add((prop_uri, RDFS.subClassOf, namespace.PropertyType))

        # Add attributes like id, code, description in English and Deutsch, property_label, data_type
        graph.add((prop_uri, RDFS.label, Literal(prop.id, lang="en")))
        graph.add((prop_uri, DC.identifier, Literal(prop.code)))
        descriptions = prop.description.split("//")
        if len(descriptions) > 1:
            graph.add((prop_uri, RDFS.comment, Literal(descriptions[0], lang="en")))
            graph.add((prop_uri, RDFS.comment, Literal(descriptions[1], lang="de")))
        else:
            graph.add((prop_uri, RDFS.comment, Literal(prop.description, lang="en")))
        graph.add(
            (prop_uri, namespace.propertyLabel, Literal(prop.property_label, lang="en"))
        )
        graph.add((prop_uri, namespace.dataType, Literal(prop.data_type.value)))
        if prop.data_type.value == "OBJECT":
            # entity_ref_uri = BAM[code_to_class_name(obj.object_code)]
            # graph.add((prop_uri, BAM.referenceTo, entity_ref_uri))
            object_code = code_to_class_name(prop.object_code, logger)
            if not object_code:
                logger.error(
                    f"Failed to identify the `object_code` for the property {prop.id}"
                )
                return prop_uri
            entity_ref_uri = namespace[object_code]

            # Create a restriction with referenceTo
            restriction = BNode()
            graph.add((restriction, RDF.type, OWL.Restriction))
            graph.add((restriction, OWL.onProperty, namespace["referenceTo"]))
            graph.add((restriction, OWL.someValuesFrom, entity_ref_uri))

            # Add the restriction as a subclass of the property
            graph.add((prop_uri, RDFS.subClassOf, restriction))
        return prop_uri

    # skos:prefLabel used for class names
    # skos:definition used for `description` (en, de)
    # dc:identifier used for `code`  # ! only defined for internal codes with $ symbol
    # parents defined from `code`
    # assigned properties can be Mandatory or Optional, can be PropertyType or ObjectType
    # ? For OBJECT TYPES
    # ? `generated_code_prefix`, `auto_generate_codes`?
    @no_type_check
    def model_to_rdf(
        self, namespace: "Namespace", graph: "Graph", logger: "BoundLoggerLazyProxy"
    ) -> None:
        """
        Convert the entity to RDF triples and add them to the graph. The function uses the
        `_add_properties_rdf` method to convert the properties assigned to the entity to RDF triples.

        Args:
            namespace (Namespace): The namespace to use for the RDF graph.
            graph (Graph): The RDF graph to which the entity is added.
            logger (BoundLoggerLazyProxy): The logger to log messages.
        """
        entity_uri = namespace[self.defs.id]

        # Define the entity as an OWL class inheriting from the specific namespace type
        graph.add((entity_uri, RDF.type, OWL.Thing))
        parent_classes = self.__class__.__bases__
        for parent_class in parent_classes:
            if issubclass(parent_class, BaseEntity) and parent_class != BaseEntity:
                # if parent_class.__name__ in [
                #     "ObjectType",
                #     "CollectionType",
                #     "DatasetType",
                # ]:
                #     # ! add here logic of subClassOf connecting with PROV-O or BFO
                #     # ! maybe via classes instead of ObjectType/CollectionType/DatasetType?
                #     # ! Example:
                #     # !     graph.add((entity_uri, RDFS.subClassOf, "http://www.w3.org/ns/prov#Entity"))
                #     continue
                parent_uri = namespace[parent_class.__name__]
                graph.add((entity_uri, RDFS.subClassOf, parent_uri))

        # Add attributes like id, code, description in English and Deutsch, property_label, data_type
        graph.add((entity_uri, RDFS.label, Literal(self.defs.id, lang="en")))
        graph.add((entity_uri, DC.identifier, Literal(self.defs.code)))
        descriptions = self.defs.description.split("//")
        if len(descriptions) > 1:
            graph.add((entity_uri, RDFS.comment, Literal(descriptions[0], lang="en")))
            graph.add((entity_uri, RDFS.comment, Literal(descriptions[1], lang="de")))
        else:
            graph.add(
                (entity_uri, RDFS.comment, Literal(self.defs.description, lang="en"))
            )
        # Adding properties relationships to the entities
        for assigned_prop in self._base_attrs:
            prop_uri = self._add_properties_rdf(namespace, graph, assigned_prop, logger)
            restriction = BNode()
            graph.add((restriction, RDF.type, OWL.Restriction))
            if assigned_prop.mandatory:
                graph.add(
                    (restriction, OWL.onProperty, namespace["hasMandatoryProperty"])
                )
            else:
                graph.add(
                    (restriction, OWL.onProperty, namespace["hasOptionalProperty"])
                )
            graph.add((restriction, OWL.someValuesFrom, prop_uri))

            # Add the restriction as a subclass of the entity
            graph.add((entity_uri, RDFS.subClassOf, restriction))

cls_name property

Returns the entity name of the class as a string to speed up checks. This is a property to be overwritten by each of the abstract entity types.

get_property_metadata()

Dictionary containing the metadata of the properties assigned to the entity type.

Returns:

Name Type Description
dict dict

A dictionary containing the keys of the PropertyTypeAssignment attribute names and the

dict

values of the definitions of PropertyTypeAssignment. Example:

dict

{ "name": PropertyTypeAssignment( code="$NAME", data_type=VARCHAR, mandatory=True, property_label="Name" ), "age": PropertyTypeAssignment( code="AGE", data_type=INTEGER, mandatory=False, property_label="Age" ),

dict

}

Source code in bam_masterdata/metadata/entities.py
def get_property_metadata(self) -> dict:
    """
    Dictionary containing the metadata of the properties assigned to the entity type.

    Returns:
        dict: A dictionary containing the keys of the `PropertyTypeAssignment` attribute names and the
        values of the definitions of `PropertyTypeAssignment`. Example:
        {
            "name": PropertyTypeAssignment(
                code="$NAME",
                data_type=VARCHAR,
                mandatory=True,
                property_label="Name"
            ),
            "age": PropertyTypeAssignment(
                code="AGE",
                data_type=INTEGER,
                mandatory=False,
                property_label="Age"
            ),
        }
    """
    cls_attrs = self.__class__.__dict__

    # Store property metadata at class level
    prop_meta_dict: dict = {}
    for base in type(self).__mro__:
        cls_attrs = getattr(base, "__dict__", {})
        for attr_name, attr_value in cls_attrs.items():
            if isinstance(attr_value, PropertyTypeAssignment):
                prop_meta_dict[attr_name] = attr_value
    return prop_meta_dict

model_to_dict()

Returns the model as a dictionary storing the data defs and the property or vocabulary term assignments.

Returns:

Name Type Description
dict dict

The dictionary representation of the model.

Source code in bam_masterdata/metadata/entities.py
def model_to_dict(self) -> dict:
    """
    Returns the model as a dictionary storing the data `defs` and the property or vocabulary term
    assignments.

    Returns:
        dict: The dictionary representation of the model.
    """
    data = self.model_dump()

    attr_value = getattr(self, "defs")
    if isinstance(attr_value, BaseModel):
        data["defs"] = attr_value.model_dump()
    else:
        data["defs"] = attr_value
    return data

model_to_json(indent=None)

Returns the model as a string in JSON format storing the data defs and the property or vocabulary term assignments.

Parameters:

Name Type Description Default
indent Optional[int]

The indent to print in JSON. Defaults to None.

None

Returns:

Name Type Description
str str

The JSON representation of the model.

Source code in bam_masterdata/metadata/entities.py
def model_to_json(self, indent: int | None = None) -> str:
    """
    Returns the model as a string in JSON format storing the data `defs` and the property or
    vocabulary term assignments.

    Args:
        indent (Optional[int], optional): The indent to print in JSON. Defaults to None.

    Returns:
        str: The JSON representation of the model.
    """
    # * `model_dump_json()` from pydantic does not store the `defs` section of each entity.
    data = self.model_to_dict()
    return json.dumps(data, indent=indent)

model_to_rdf(namespace, graph, logger)

Convert the entity to RDF triples and add them to the graph. The function uses the _add_properties_rdf method to convert the properties assigned to the entity to RDF triples.

Parameters:

Name Type Description Default
namespace Namespace

The namespace to use for the RDF graph.

required
graph Graph

The RDF graph to which the entity is added.

required
logger BoundLoggerLazyProxy

The logger to log messages.

required
Source code in bam_masterdata/metadata/entities.py
@no_type_check
def model_to_rdf(
    self, namespace: "Namespace", graph: "Graph", logger: "BoundLoggerLazyProxy"
) -> None:
    """
    Convert the entity to RDF triples and add them to the graph. The function uses the
    `_add_properties_rdf` method to convert the properties assigned to the entity to RDF triples.

    Args:
        namespace (Namespace): The namespace to use for the RDF graph.
        graph (Graph): The RDF graph to which the entity is added.
        logger (BoundLoggerLazyProxy): The logger to log messages.
    """
    entity_uri = namespace[self.defs.id]

    # Define the entity as an OWL class inheriting from the specific namespace type
    graph.add((entity_uri, RDF.type, OWL.Thing))
    parent_classes = self.__class__.__bases__
    for parent_class in parent_classes:
        if issubclass(parent_class, BaseEntity) and parent_class != BaseEntity:
            # if parent_class.__name__ in [
            #     "ObjectType",
            #     "CollectionType",
            #     "DatasetType",
            # ]:
            #     # ! add here logic of subClassOf connecting with PROV-O or BFO
            #     # ! maybe via classes instead of ObjectType/CollectionType/DatasetType?
            #     # ! Example:
            #     # !     graph.add((entity_uri, RDFS.subClassOf, "http://www.w3.org/ns/prov#Entity"))
            #     continue
            parent_uri = namespace[parent_class.__name__]
            graph.add((entity_uri, RDFS.subClassOf, parent_uri))

    # Add attributes like id, code, description in English and Deutsch, property_label, data_type
    graph.add((entity_uri, RDFS.label, Literal(self.defs.id, lang="en")))
    graph.add((entity_uri, DC.identifier, Literal(self.defs.code)))
    descriptions = self.defs.description.split("//")
    if len(descriptions) > 1:
        graph.add((entity_uri, RDFS.comment, Literal(descriptions[0], lang="en")))
        graph.add((entity_uri, RDFS.comment, Literal(descriptions[1], lang="de")))
    else:
        graph.add(
            (entity_uri, RDFS.comment, Literal(self.defs.description, lang="en"))
        )
    # Adding properties relationships to the entities
    for assigned_prop in self._base_attrs:
        prop_uri = self._add_properties_rdf(namespace, graph, assigned_prop, logger)
        restriction = BNode()
        graph.add((restriction, RDF.type, OWL.Restriction))
        if assigned_prop.mandatory:
            graph.add(
                (restriction, OWL.onProperty, namespace["hasMandatoryProperty"])
            )
        else:
            graph.add(
                (restriction, OWL.onProperty, namespace["hasOptionalProperty"])
            )
        graph.add((restriction, OWL.someValuesFrom, prop_uri))

        # Add the restriction as a subclass of the entity
        graph.add((entity_uri, RDFS.subClassOf, restriction))

to_dict()

Returns the entity as a dictionary storing the value of the properties assigned to the entity.

Returns:

Name Type Description
dict dict

The dictionary representation of the entity.

Source code in bam_masterdata/metadata/entities.py
def to_dict(self) -> dict:
    """
    Returns the entity as a dictionary storing the value of the properties assigned to the entity.

    Returns:
        dict: The dictionary representation of the entity.
    """
    dump_json = self.to_json()
    return json.loads(dump_json)

to_hdf5(hdf_file, group_name='')

Serialize the entity to a HDF5 file under the group specified in the input.

Parameters:

Name Type Description Default
hdf_file File

The HDF5 file to store the entity.

required
group_name str

The group name to serialize the data.

''
Source code in bam_masterdata/metadata/entities.py
def to_hdf5(self, hdf_file: h5py.File, group_name: str = "") -> h5py.File:
    """
    Serialize the entity to a HDF5 file under the group specified in the input.

    Args:
        hdf_file (h5py.File): The HDF5 file to store the entity.
        group_name (str, optional): The group name to serialize the data.
    """
    if not group_name:
        group_name = self.cls_name
    group = hdf_file.create_group(group_name)

    for key in self._property_metadata.keys():
        try:
            value = getattr(self, key)
            if not value:
                continue
            if isinstance(value, str | int | float | bool | list | tuple):
                group.create_dataset(key, data=value)
            else:
                raise TypeError(
                    f"Unsupported type {type(value)} for key {key} for HDF5 serialization."
                )
        except AttributeError:
            continue

to_json(indent=None)

Returns the entity as a string in JSON format storing the value of the properties assigned to the entity.

Parameters:

Name Type Description Default
indent Optional[int]

The indent to print in JSON. Defaults to None.

None

Returns:

Name Type Description
str str

The JSON representation of the entity.

Source code in bam_masterdata/metadata/entities.py
def to_json(self, indent: int | None = None) -> str:
    """
    Returns the entity as a string in JSON format storing the value of the properties
    assigned to the entity.

    Args:
        indent (Optional[int], optional): The indent to print in JSON. Defaults to None.

    Returns:
        str: The JSON representation of the entity.
    """
    data: dict = {}
    for key in self._property_metadata.keys():
        try:
            value = getattr(self, key)
        except AttributeError:
            continue
        if isinstance(value, PropertyTypeAssignment):
            continue
        data[key] = value
    return json.dumps(data, indent=indent, sort_keys=True)

ObjectType

Bases: BaseEntity

Base class used to define object types. All object types must inherit from this class. The object types are defined in the module bam_masterdata/object_types.py.

The ObjectType class contains a list of all properties defined for a ObjectType, for internally represent the model in other formats (e.g., JSON or Excel).

Note this is also used for CollectionType and DatasetType, as they also contain a list of properties.

Source code in bam_masterdata/metadata/entities.py
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
class ObjectType(BaseEntity):
    """
    Base class used to define object types. All object types must inherit from this class. The
    object types are defined in the module `bam_masterdata/object_types.py`.

    The `ObjectType` class contains a list of all `properties` defined for a `ObjectType`, for
    internally represent the model in other formats (e.g., JSON or Excel).

    Note this is also used for `CollectionType` and `DatasetType`, as they also contain a list of
    properties.
    """

    model_config = ConfigDict(
        ignored_types=(
            ObjectTypeDef,
            CollectionTypeDef,
            DatasetTypeDef,
            PropertyTypeAssignment,
        )
    )

    properties: list[PropertyTypeAssignment] = Field(
        default_factory=list,
        description="""
        List of properties assigned to an object type. This is useful for internal representation of the model.
        """,
    )

    datasets: list[str] = Field(
        default_factory=list,
        exclude=True,
        description="""
        List of file paths attached to this object as datasets. Useful for attaching data or plots directly to objects.
        """,
    )

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        # Initialize the properties list to store PropertyTypeAssignment instances
        self._properties = {}
        for key, prop in self._property_metadata.items():
            self._properties[key] = prop.data_type

    def _set_object_value(self, key, value):
        """
        Sets the value when the data type is OBJECT.
        """
        if isinstance(value, str):
            # Validate the path format: /{space}/{project}/{collection}/{object} or /{space}/{project}/{object}
            # If path is valid, store it as-is
            if not value.startswith("/"):
                raise ValueError(
                    f"Invalid OBJECT path format for '{key}': Path must start with '/', got '{value}'"
                )
            path_parts = value.strip("/").split("/")
            if len(path_parts) not in [3, 4]:
                raise ValueError(
                    f"Invalid OBJECT path format for '{key}': Expected '/<space>/<project>/<collection>/<object>' "
                    f"or '/<space>/<project>/<object>', got '{value}'"
                )
            # * We don't validate if the object exists here as it requires pybis connection
            # * That validation should be done when saving to openBIS
        elif isinstance(value, ObjectType):
            # Check if the object instance has a code
            if not hasattr(value, "code") or value.code is None:
                raise ValueError(
                    f"Object instance for '{key}' must have a 'code' attribute set to be used as a reference"
                )
        else:
            raise TypeError(
                f"Invalid type for OBJECT property '{key}': Expected str (path) or ObjectType instance, "
                f"got {type(value).__name__}"
            )
        return value

    def _validate_controlled_vocabulary(self, meta, key, value) -> None:
        """
        Validates the value of a CONTROLLEDVOCABULARY.
        """
        vocabulary_code = meta[key].vocabulary_code

        # Patch to handle institutional vocabularies
        if vocabulary_code in [
            "BAM_FLOOR",
            "BAM_HOUSE",
            "BAM_LOCATION",
            "BAM_LOCATION_COMPLETE",
            "BAM_OE",
            "BAM_ROOM",
            "PERSON_STATUS",
        ]:
            warnings.warn(
                f"The attribute '{key}' uses the institutional vocabulary '{vocabulary_code}'. "
                "This value will not be validated against internal vocabulary definitions.",
                UserWarning,
                stacklevel=3,
            )
            return None

        if not vocabulary_code:
            raise ValueError(
                f"Property '{key}' of type CONTROLLEDVOCABULARY must have a vocabulary_code defined."
            )
        vocab_paths = []
        for file in listdir_py_modules(DATAMODEL_DIR):
            if "vocabulary_types.py" in file or "vocabularies.py" in file:
                vocab_paths.append(file)
        if not vocab_paths:
            raise FileNotFoundError(
                f"No file 'vocabularies.py' was not found in the directory and sub-directories specified by {DATAMODEL_DIR}."
            )

        # Scan all vocabularies.py files
        vocabulary_class = None
        for vocab_path in vocab_paths:
            vocabulary_class = self.get_vocabulary_class(vocabulary_code, vocab_path)
            if vocabulary_class is not None:
                break
        if vocabulary_class is None:
            raise ValueError(
                f"No matching vocabulary class found for vocabulary_code '{vocabulary_code}'."
            )
        codes = [term.code for term in vocabulary_class.terms]
        if value not in codes:
            raise ValueError(
                f"{value} for {key} is not in the list of allowed terms for vocabulary."
            )

    def __setattr__(self, key, value):
        if not hasattr(self, "_property_metadata"):
            return object.__setattr__(self, key, value)

        if key in [
            "_property_metadata",
            "_properties",
            "code",
            "properties",
            "datasets",
        ]:
            super().__setattr__(key, value)
            return

        # key search in every nested class
        for base in type(self).__mro__:
            prop_meta = getattr(base, "get_property_metadata", None)
            if callable(prop_meta):
                meta = (
                    prop_meta(self)
                    if base is not type(self)
                    else self._property_metadata
                )
                if key in meta:
                    # Type check
                    expected_type = meta[key].data_type.pytype
                    if expected_type is datetime.datetime:
                        if isinstance(value, datetime.datetime):
                            try:
                                value = value.strftime(
                                    "%Y-%m-%d %H:%M:%S"
                                )  # create string
                                expected_type = str
                            except ValueError:
                                raise ValueError(
                                    f"Invalid datetime format for '{key}': Expected ISO format string, got '{value}'"
                                )
                        elif isinstance(value, str):
                            try:
                                datetime.datetime.fromisoformat(value)
                                expected_type = str
                            except ValueError:
                                raise ValueError(
                                    f"Invalid datetime format for '{key}': Expected ISO format string, got '{value}'"
                                )
                        else:
                            raise TypeError(
                                f"Invalid type for '{key}': Expected datetime or ISO format string, got {type(value).__name__}"
                            )
                    if expected_type and not isinstance(value, expected_type):
                        raise TypeError(
                            f"Invalid type for '{key}': Expected {expected_type.__name__}, got {type(value).__name__}"
                        )

                    # Get data type for additional checks
                    data_type = meta[key].data_type
                    # OBJECT check and attr assignment
                    if data_type == "OBJECT":
                        return object.__setattr__(
                            self, key, self._set_object_value(key, value)
                        )
                    # CONTROLLEDVOCABULARY check
                    if data_type == "CONTROLLEDVOCABULARY":
                        self._validate_controlled_vocabulary(meta, key, value)

                    # Setting attribute value after all checks
                    return object.__setattr__(self, key, value)

        raise KeyError(
            f"Key '{key}' not found in any property_metadata of {type(self).__name__} or its bases."
        )

    def get_vocabulary_class(
        self, vocabulary_code: str, vocab_path: str
    ) -> VocabularyType | None:
        """
        Get the class instance of the vocabulary type defined by `vocabulary_code` in the Python module
        specified by `vocab_path`.

        Args:
            vocabulary_code (str): Code of the vocabulary type to get.
            vocab_path (str): Path to the module containing the vocabulary type definitions.

        Returns:
            VocabularyType | None: The class of the vocabulary type if found, otherwise None.
        """
        module = import_module(vocab_path)
        vocabulary_class = None
        for _, obj in inspect.getmembers(module, inspect.isclass):
            if obj.__module__ != module.__name__:
                continue

            if obj.defs.code == vocabulary_code:
                vocabulary_class = obj()
                break

        return vocabulary_class

    @property
    def base_name(self) -> str:
        """
        Returns the entity name of the class as a string.
        """
        return "ObjectType"

    @model_validator(mode="after")
    @classmethod
    def model_validator_after_init(cls, data: Any) -> Any:
        """
        Validate the model after instantiation of the class.

        Args:
            data (Any): The data containing the fields values to validate.

        Returns:
            Any: The data with the validated fields.
        """
        # ordered parent -> child properties
        collected_properties = []
        for base in reversed(cls.__mro__):
            for _, attr_val in base.__dict__.items():
                if isinstance(attr_val, PropertyTypeAssignment):
                    collected_properties.append(attr_val)

        # Group by section preserving first appearance order
        grouped_sections = OrderedDict()
        for prop in collected_properties:
            section = prop.section or ""
            grouped_sections.setdefault(section, [])
            grouped_sections[section].append(prop)

        # Flatten grouped properties
        ordered_properties = []
        for props in grouped_sections.values():
            ordered_properties.extend(props)
        data.properties = ordered_properties

        return data

    @deprecated(
        "This method will soon be deprecated and will be removed in a future major release. "
        "The push to openbis and synchronization happens in `to_openbis_sync()` instead."
    )
    def to_openbis(self, logger: "BoundLoggerLazyProxy", openbis: "Openbis") -> None:
        def _assign_property(prop, entity, openbis) -> None:
            """
            Assign the property to the entity, adding the `vocabulary` parameter if the `vocabulary_code`
            is defined for the property. If the property does not exist in openBIS, create it before assigning to the entity.

            Args:
                prop (PropertyTypeAssignment): The property to assign.
                entity (Entity): The entity to assign the property to.
                openbis (Openbis): The openBIS instance.
            """
            # Handle special case for OBJECT data type to map into the legacy name
            if prop.data_type == "OBJECT":
                prop.data_type = DataType.SAMPLE

            # If property does not exist in openBIS, create it before assigning to the entity
            if not openbis.get_property_types(prop.code):
                # For CONTROLLEDVOCABULARY properties with a defined vocabulary_code, we need to create the property with the vocabulary assigned
                if prop.data_type == "CONTROLLEDVOCABULARY" and prop.vocabulary_code:
                    openbis.new_property_type(
                        code=prop.code,
                        label=prop.property_label,
                        description=prop.description,
                        dataType=prop.data_type.value,
                        vocabulary=prop.vocabulary_code,
                    ).save()
                else:
                    openbis.new_property_type(
                        code=prop.code,
                        label=prop.property_label,
                        description=prop.description,
                        dataType=prop.data_type.value,
                    ).save()

            # Assign the property to the entity, adding the `vocabulary` parameter if the `vocabulary_code` is defined for the property
            # TODO check what happens when dataType="SAMPLE" and how to use pyBIS for this
            if prop.vocabulary_code:
                entity.assign_property(
                    prop=prop.code,
                    section=prop.section,
                    mandatory=prop.mandatory,
                    showInEditView=prop.show_in_edit_views,
                    ordinal=prop.ordinal,
                )
            else:
                entity.assign_property(
                    prop=prop.code,
                    section=prop.section,
                    mandatory=prop.mandatory,
                    showInEditView=prop.show_in_edit_views,
                    ordinal=prop.ordinal,
                )

        # Get all existing entities from openBIS
        openbis_entities = OpenbisEntities(url=openbis.url).get_object_dict()

        if self.defs.code in openbis_entities:
            logger.info(f"Object type '{self.defs.code}' already exists in openBIS.")
            entity = openbis.get_object_type(self.defs.code)

            # Get properties from openBIS
            obis_properties = entity.get_property_assignments()
            obis_property_codes = [prop.code for prop in obis_properties]

            # Check for properties in self that are not in openBIS
            new_properties_added = False
            for prop in self.properties:
                if prop.code not in obis_property_codes:
                    logger.info(
                        f"Adding new property '{prop.code}' to object type '{self.defs.code}'."
                    )
                    new_properties_added = True
                    _assign_property(prop, entity, openbis)

            if not new_properties_added:
                logger.info(
                    f"No new properties added to object type '{self.defs.code}'."
                )

        else:
            logger.info(f"Creating new object type '{self.defs.code}' in openBIS.")
            entity = openbis.new_object_type(
                code=self.defs.code,
                description=self.defs.description,
                validationPlugin=self.defs.validation_script,
                generatedCodePrefix=self.defs.generated_code_prefix,
                autoGeneratedCode=self.defs.auto_generate_codes,
            )
            entity.save()  # we need to save this before assigning properties

            # Assign properties to the new entity
            for prop in self.properties:
                logger.info(
                    f"Adding new property '{prop.code}' to object type '{self.defs.code}'."
                )
                _assign_property(prop, entity, openbis)

        entity.save()
        return entity

    def dataset(self):
        # nur lesbar von außen
        return self.datasets

    def add_dataset(self, file):
        self.datasets.append(file)

    def _openbis_data_type(self, prop: PropertyTypeAssignment) -> str:
        """Translates bam-masterdata data types to their openBIS representation.

        Args:
            prop (PropertyTypeAssignment): The property to translate its data type.

        Returns:
            str: The value string as defined in `DataType`.
        """
        if prop.data_type in (DataType.OBJECT, DataType.SAMPLE):
            return DataType.SAMPLE.value
        return prop.data_type.value

    def _create_object_type(self, openbis: "Openbis", result: SyncResult) -> None:
        """
        Creates a new object type using the `new_object_type()` method from pyBIS.

        Args:
            openbis (Openbis): The openBIS instance to create the object type.
            result (SyncResult): The result metadata of the synchronization process.
        """
        obj_type = openbis.new_object_type(
            code=self.defs.code,
            description=self.defs.description,
            validationPlugin=self.defs.validation_script,
            generatedCodePrefix=self.defs.generated_code_prefix,
            autoGeneratedCode=self.defs.auto_generate_codes,
        )
        obj_type.save()

        result.created(
            target=SyncTarget.OBJECT_TYPE,
            code=self.defs.code,
        )

        # Synchronizing properties
        self._sync_property_assignments(
            obj_type=obj_type, openbis=openbis, result=result
        )

    def _sync_object_type_definition(
        self,
        obj_type: "SampleType",
        result: SyncResult,
    ) -> None:
        """
        Synchronizes an object type definition in the openBIS instance.

        Args:
            obj_type (SampleType): The object type to be synchronized.
            result (SyncResult): The result metadata of the synchronization process.
        """
        if obj_type.description != self.defs.description:
            old_value = obj_type.description
            obj_type.description = self.defs.description

            result.modified(
                target=SyncTarget.OBJECT_TYPE,
                code=self.defs.code,
                field="description",
                old_value=old_value,
                new_value=self.defs.description,
            )

        if obj_type.generatedCodePrefix != self.defs.generated_code_prefix:
            result.rejected(
                target=SyncTarget.OBJECT_TYPE,
                code=self.defs.code,
                field="generated_code_prefix",
                old_value=obj_type.generatedCodePrefix,
                new_value=self.defs.generated_code_prefix,
            )

    def _update_existing_property(
        self,
        prop: PropertyTypeAssignment,
        obj_type: "SampleType",
        openbis: "Openbis",
        result: SyncResult,
    ) -> None:
        """
        Updates an existing property type and its assignment metadata.

        Args:
            prop (PropertyTypeAssignment): The property type to be updated.
            obj_type (SampleType): The object type where the property is assigned to.
            openbis (Openbis): The openBIS instance where entities exists.
            result (SyncResult): The result metadata of the synchronization process.
        """
        if prop.code.startswith("$"):
            result.add(
                action=SyncAction.UNCHANGED,
                target=SyncTarget.PROPERTY_ASSIGNMENT,
                code=prop.code,
                parent_code=self.defs.code,
                field="type_and_assignment",
                old_value=None,
                new_value=None,
                message="Internal property types and assignments cannot be modified by the current user.",
            )
            return None

        # --- PropertyType fields ---
        property_type = openbis.get_property_type(prop.code)
        expected_data_type = self._openbis_data_type(prop)
        # Unsupported: data type changes
        if property_type.dataType != expected_data_type:
            result.rejected(
                target=SyncTarget.PROPERTY_TYPE,
                code=prop.code,
                field="data_type",
                old_value=property_type.dataType,
                new_value=expected_data_type,
            )
            return None

        # Unsupported: vocabulary changes
        if prop.data_type == DataType.CONTROLLEDVOCABULARY:
            current_vocabulary = getattr(property_type, "vocabulary", None)

            if current_vocabulary != prop.vocabulary_code:
                result.rejected(
                    target=SyncTarget.PROPERTY_TYPE,
                    code=prop.code,
                    field="vocabulary_code",
                    old_value=current_vocabulary,
                    new_value=prop.vocabulary_code,
                )
                return None

        property_type_modified = False

        # Supported: label
        if property_type.label != prop.property_label:
            old_value = property_type.label
            property_type.label = prop.property_label
            property_type_modified = True

            result.modified(
                target=SyncTarget.PROPERTY_TYPE,
                code=prop.code,
                field="property_label",
                old_value=old_value,
                new_value=prop.property_label,
            )

        # Supported: description
        if property_type.description != prop.description:
            old_value = property_type.description
            property_type.description = prop.description
            property_type_modified = True

            result.modified(
                target=SyncTarget.PROPERTY_TYPE,
                code=prop.code,
                field="description",
                old_value=old_value,
                new_value=prop.description,
            )

        if property_type_modified:
            property_type.save()

        # --- PropertyAssignment fields ---

        assignments = obj_type.get_property_assignments()
        assignment = next(
            assignment for assignment in assignments if assignment.code == prop.code
        )

        assignment_modified = False

        if assignment.section != prop.section:
            result.modified(
                target=SyncTarget.PROPERTY_ASSIGNMENT,
                code=prop.code,
                parent_code=self.defs.code,
                field="section",
                old_value=assignment.section,
                new_value=prop.section,
            )
            assignment_modified = True

        if assignment.mandatory != prop.mandatory:
            result.modified(
                target=SyncTarget.PROPERTY_ASSIGNMENT,
                code=prop.code,
                parent_code=self.defs.code,
                field="mandatory",
                old_value=assignment.mandatory,
                new_value=prop.mandatory,
            )
            assignment_modified = True

        if assignment.showInEditView != prop.show_in_edit_views:
            result.modified(
                target=SyncTarget.PROPERTY_ASSIGNMENT,
                code=prop.code,
                parent_code=self.defs.code,
                field="show_in_edit_views",
                old_value=assignment.showInEditView,
                new_value=prop.show_in_edit_views,
            )
            assignment_modified = True

        if assignment.ordinal != prop.ordinal:
            result.add(
                action=SyncAction.IGNORED,
                target=SyncTarget.PROPERTY_ASSIGNMENT,
                code=prop.code,
                parent_code=self.defs.code,
                message="Modification of the ordinal attribute in property assignments is not supported yet.",
            )

        if assignment_modified:
            obj_type.assign_property(
                prop=prop.code,
                section=prop.section,
                mandatory=prop.mandatory,
                showInEditView=prop.show_in_edit_views,
                ordinal=prop.ordinal,
            )

    def _create_property_assignment(
        self,
        prop: PropertyTypeAssignment,
        obj_type: "SampleType",
        openbis: "Openbis",
        result: SyncResult,
    ) -> None:
        """
        Creates (optionally, if property type does not exist already in openBIS) and assigns
        the property `prop` to the object type `obj_type`.

        Args:
            prop (PropertyTypeAssignment): The property type to be (optionally) created and assigned.
            obj_type (SampleType): The object type where the property is assigned to.
            openbis (Openbis): The openBIS instance where the entity definitions exists.
            result (SyncResult): The result metadata of the synchronization process.
        """
        try:
            _ = openbis.get_property_type(prop.code)
        except ValueError:
            kwargs = {
                "code": prop.code,
                "label": prop.property_label,
                "description": prop.description,
                "dataType": self._openbis_data_type(prop),
            }
            # For CONTROLLEDVOCABULARY properties with a defined vocabulary_code, we need to create the property with the vocabulary assigned
            if prop.data_type == DataType.CONTROLLEDVOCABULARY and prop.vocabulary_code:
                kwargs["vocabulary"] = prop.vocabulary_code

            openbis.new_property_type(**kwargs).save()
            result.created(
                target=SyncTarget.PROPERTY_TYPE,
                code=prop.code,
            )

        obj_type.assign_property(
            prop=prop.code,
            section=prop.section,
            mandatory=prop.mandatory,
            showInEditView=prop.show_in_edit_views,
            ordinal=prop.ordinal,
        )

        result.created(
            target=SyncTarget.PROPERTY_ASSIGNMENT,
            code=prop.code,
            parent_code=self.defs.code,
        )

    def _sync_property_assignments(
        self,
        obj_type: "SampleType",
        openbis: "Openbis",
        result: SyncResult,
    ) -> None:
        """
        Synchronizes the property assignemtns to the object type specified in `obj_type`.

        Args:
            obj_type (SampleType): The object type where properties are assigned and synchronized.
            openbis (Openbis): The openBIS instance where the defintions (both of object type and property type) exists.
            result (SyncResult): The result metadata of the synchronization process.
        """
        # openBIS property assigned codes
        openbis_assignments = obj_type.get_property_assignments()
        openbis_codes = set(openbis_assignments.df["code"])
        # BAM Masterdata property assigned codes
        model_props = {prop.code: prop for prop in self.properties}
        model_codes = set(model_props)

        # Comparison
        # Preserve the order defined in self.properties
        new_codes = [
            prop.code for prop in self.properties if prop.code not in openbis_codes
        ]

        existing_codes = [
            prop.code for prop in self.properties if prop.code in openbis_codes
        ]
        # Order here is less important because these are only reported as missing.
        missing_codes = openbis_codes - model_codes

        # Creating and assigning properties
        for code in new_codes:
            self._create_property_assignment(
                prop=model_props[code],
                obj_type=obj_type,
                openbis=openbis,
                result=result,
            )

        # Modifying existing assigned properties
        for code in existing_codes:
            self._update_existing_property(
                prop=model_props[code],
                obj_type=obj_type,
                openbis=openbis,
                result=result,
            )

        # Ignoring deletion of assigned properties
        for code in missing_codes:
            result.add(
                action=SyncAction.IGNORED,
                target=SyncTarget.PROPERTY_ASSIGNMENT,
                code=code,
                parent_code=self.defs.code,
                message="Deletion of property assignments is not implemented yet.",
            )

    def to_openbis_sync(self, openbis: "Openbis") -> SyncResult:
        """
        Synchronizes this ObjectType definition and its PropertyTypeAssignment with the `openbis` instance.

        Supported operations:
        - create new object types
        - add new assigned properties to existing object types
        - modify specific fields in object types or assigned properties

        Not supported operations:
        - deletion of object types
        - deletion of previously assigned properties
        - assigned properties code changes
        - assigned properties unit changes
        - assigned properties data type changes

        Args:
            logger (BoundLoggerLazyProxy): The logger to log messages.
            openbis (Openbis): The openBIS instance to synchronize this object type to.

        Returns:
            (SyncResult): The result of the synchronization process. Useful for structuring logging.
        """
        result = SyncResult()

        # Get all existing object types from openBIS
        openbis_object_types = OpenbisEntities(url=openbis).get_object_dict()

        # Create object type if missing in openBIS
        if self.defs.code not in openbis_object_types:
            self._create_object_type(openbis=openbis, result=result)
            return result

        obj_type = openbis.get_object_type(self.defs.code)

        # Otherwise, synchronize definition
        self._sync_object_type_definition(obj_type=obj_type, result=result)
        # Then synchronize properties
        self._sync_property_assignments(
            obj_type=obj_type, openbis=openbis, result=result
        )

        obj_type.save()

        return result

base_name property

Returns the entity name of the class as a string.

get_vocabulary_class(vocabulary_code, vocab_path)

Get the class instance of the vocabulary type defined by vocabulary_code in the Python module specified by vocab_path.

Parameters:

Name Type Description Default
vocabulary_code str

Code of the vocabulary type to get.

required
vocab_path str

Path to the module containing the vocabulary type definitions.

required

Returns:

Type Description
VocabularyType | None

VocabularyType | None: The class of the vocabulary type if found, otherwise None.

Source code in bam_masterdata/metadata/entities.py
def get_vocabulary_class(
    self, vocabulary_code: str, vocab_path: str
) -> VocabularyType | None:
    """
    Get the class instance of the vocabulary type defined by `vocabulary_code` in the Python module
    specified by `vocab_path`.

    Args:
        vocabulary_code (str): Code of the vocabulary type to get.
        vocab_path (str): Path to the module containing the vocabulary type definitions.

    Returns:
        VocabularyType | None: The class of the vocabulary type if found, otherwise None.
    """
    module = import_module(vocab_path)
    vocabulary_class = None
    for _, obj in inspect.getmembers(module, inspect.isclass):
        if obj.__module__ != module.__name__:
            continue

        if obj.defs.code == vocabulary_code:
            vocabulary_class = obj()
            break

    return vocabulary_class

model_validator_after_init(data) classmethod

Validate the model after instantiation of the class.

Parameters:

Name Type Description Default
data Any

The data containing the fields values to validate.

required

Returns:

Name Type Description
Any Any

The data with the validated fields.

Source code in bam_masterdata/metadata/entities.py
@model_validator(mode="after")
@classmethod
def model_validator_after_init(cls, data: Any) -> Any:
    """
    Validate the model after instantiation of the class.

    Args:
        data (Any): The data containing the fields values to validate.

    Returns:
        Any: The data with the validated fields.
    """
    # ordered parent -> child properties
    collected_properties = []
    for base in reversed(cls.__mro__):
        for _, attr_val in base.__dict__.items():
            if isinstance(attr_val, PropertyTypeAssignment):
                collected_properties.append(attr_val)

    # Group by section preserving first appearance order
    grouped_sections = OrderedDict()
    for prop in collected_properties:
        section = prop.section or ""
        grouped_sections.setdefault(section, [])
        grouped_sections[section].append(prop)

    # Flatten grouped properties
    ordered_properties = []
    for props in grouped_sections.values():
        ordered_properties.extend(props)
    data.properties = ordered_properties

    return data

to_openbis_sync(openbis)

Synchronizes this ObjectType definition and its PropertyTypeAssignment with the openbis instance.

Supported operations: - create new object types - add new assigned properties to existing object types - modify specific fields in object types or assigned properties

Not supported operations: - deletion of object types - deletion of previously assigned properties - assigned properties code changes - assigned properties unit changes - assigned properties data type changes

Parameters:

Name Type Description Default
logger BoundLoggerLazyProxy

The logger to log messages.

required
openbis Openbis

The openBIS instance to synchronize this object type to.

required

Returns:

Type Description
SyncResult

The result of the synchronization process. Useful for structuring logging.

Source code in bam_masterdata/metadata/entities.py
def to_openbis_sync(self, openbis: "Openbis") -> SyncResult:
    """
    Synchronizes this ObjectType definition and its PropertyTypeAssignment with the `openbis` instance.

    Supported operations:
    - create new object types
    - add new assigned properties to existing object types
    - modify specific fields in object types or assigned properties

    Not supported operations:
    - deletion of object types
    - deletion of previously assigned properties
    - assigned properties code changes
    - assigned properties unit changes
    - assigned properties data type changes

    Args:
        logger (BoundLoggerLazyProxy): The logger to log messages.
        openbis (Openbis): The openBIS instance to synchronize this object type to.

    Returns:
        (SyncResult): The result of the synchronization process. Useful for structuring logging.
    """
    result = SyncResult()

    # Get all existing object types from openBIS
    openbis_object_types = OpenbisEntities(url=openbis).get_object_dict()

    # Create object type if missing in openBIS
    if self.defs.code not in openbis_object_types:
        self._create_object_type(openbis=openbis, result=result)
        return result

    obj_type = openbis.get_object_type(self.defs.code)

    # Otherwise, synchronize definition
    self._sync_object_type_definition(obj_type=obj_type, result=result)
    # Then synchronize properties
    self._sync_property_assignments(
        obj_type=obj_type, openbis=openbis, result=result
    )

    obj_type.save()

    return result

CollectionType

Bases: ObjectType

Source code in bam_masterdata/metadata/entities.py
class CollectionType(ObjectType):
    model_config = ConfigDict(
        ignored_types=(
            ObjectTypeDef,
            ObjectType,
            CollectionTypeDef,
            PropertyTypeAssignment,
        )
    )

    attached_objects: dict[str, ObjectType] = Field(
        default_factory=dict,
        exclude=True,
        description="""
        Dictionary containing the object types attached to the collection type.
        The keys are object unique identifiers and the values are the ObjectType instances.
        """,
    )

    destinations: dict[str, Destination] = Field(
        default_factory=dict,
        exclude=True,
        description="""
        Dictionary containing destination overrides for objects attached to the collection type.

        The keys correspond to object identifiers in `attached_objects`. Objects without an entry
        use the default destination configured in `RunParsers`, e.g.:

        ```python
        attached_objects = {
            "OBJ123": sample,
            "OBJ456": step,
        }

        destinations = {
            "OBJ123": Destination(
                project="OTHER_PROJECT",
            ),
        }
        ```
        """,
    )

    relationships: dict[str, tuple[str, str]] = Field(
        default_factory=dict,
        exclude=True,
        description="""
        Dictionary containing the relationships between the objects attached to the collection type.
        The keys are relationships unique identifiers, the values are the object unique identifiers as a
        tuple, and the order is always (parent_id, child_id).
        """,
    )

    def __repr__(self):
        return (
            f"{self.base_name}("
            f"attached_objects={self.attached_objects}, "
            f"destinations={self.destinations}, "
            f"relationships={self.relationships}"
            f")"
        )

    @property
    def base_name(self) -> str:
        """
        Returns the entity name of the class as a string.
        """
        return "CollectionType"

    @deprecated(
        "This method is deprecated and will be removed in a future major release. The creation of Object Types, "
        "Property Types, and Vocabularies is done now in the corresponding ObjectType and VocabularyType classes."
    )
    def to_openbis(
        self,
        logger: "BoundLoggerLazyProxy",
        openbis: "Openbis",
        type: str = "collection",
        type_map: dict = COLLECTION_TYPE_MAP,
    ) -> None:
        def get_type(openbis: "Openbis", code: str):
            return openbis.get_collection_type(code)

        def create_type(openbis: "Openbis", defs: CollectionTypeDef):
            if defs.validation_script == "None":
                defs.validation_script = None
            if defs.validation_script:
                return openbis.new_collection_type(
                    code=defs.code,
                    description=defs.description,
                    validationPlugin=defs.validation_script,
                )
            else:
                return openbis.new_collection_type(
                    code=defs.code,
                    description=defs.description,
                    validationPlugin="",
                )

        super()._to_openbis(
            logger=logger,
            openbis=openbis,
            type=type,
            type_map=type_map,
            get_type=get_type,
            create_type=create_type,
        )

    def add(
        self,
        object_type: ObjectType,
        destination: Destination | None = None,
    ) -> str:
        """
        Add an object type to the collection type.

        Args:
            object_type (ObjectType): The object type to add to the collection type.
            destination (Destination | None): Optional openBIS destination override for this object.
                If omitted, the default destination configured in RunParsers is used.

        Returns:
            str: The unique identifier of the object type assigned in openBIS.
        """
        if not isinstance(object_type, ObjectType):
            raise TypeError(
                f"Expected an ObjectType instance, got `{type(object_type).__name__}`"
            )

        # Check mandatory properties are filled
        missing_fields = []
        for attr_name, prop in object_type._property_metadata.items():
            assigned_prop = getattr(object_type, attr_name, None)
            if prop.mandatory and isinstance(assigned_prop, PropertyTypeAssignment):
                missing_fields.append(attr_name)

        if missing_fields:
            raise ValueError(
                f"The following mandatory fields are missing for ObjectType '{object_type.cls_name}': {', '.join(missing_fields)}"
            )

        object_id = generate_object_id(object_type)
        self.attached_objects[object_id] = object_type
        if destination is not None:
            self.destinations[object_id] = destination
        return object_id

    def remove(self, object_id: str = "") -> None:
        """
        Remove an object type from the collection type by its unique identifier.

        Args:
            object_id (str, optional): The ID of the object type to be removed from the collection.
        """
        if not object_id:
            raise ValueError(
                "You must provide an `object_id` to remove the object type from the collection."
            )
        if object_id not in self.attached_objects.keys():
            raise ValueError(
                f"Object with ID '{object_id}' does not exist in the collection."
            )
        del self.attached_objects[object_id]
        self.destinations.pop(object_id, None)

    def add_relationship(self, parent: str | dict, child: str | dict) -> str:
        """
        Add a relationship between two object types in the collection type.

        Args:
            parent (str | dict):
                Either the unique identifier of the parent object type or a
                dictionary representation of the parent object type.
            child (str | dict):
                Either the unique identifier of the child object type or a
                dictionary representation of the child object type.

        Returns:
            str:
                The unique identifier of the created relationship, generated
                from the resolved parent and child identifiers.
        """
        if not parent or not child:
            raise ValueError(
                "Both `parent` and `child` must be provided to add a relationship."
            )

        for obj in (parent, child):
            if not isinstance(obj, dict) and obj not in self.attached_objects:
                raise ValueError(
                    "Both `parent` and `child` must be assigned to objects attached to the collection or being a dict readable by the system."
                )

        if not isinstance(parent, dict) and not isinstance(child, dict):
            relationship_id = generate_object_relationship_id(parent, child)
            self.relationships[relationship_id] = (parent, child)
        else:
            resolved_parent = (
                generate_dict_id(parent) if isinstance(parent, dict) else parent
            )

            resolved_child = (
                generate_dict_id(child) if isinstance(child, dict) else child
            )

            relationship_id = generate_object_relationship_id(
                resolved_parent, resolved_child
            )
            self.relationships[relationship_id] = (parent, child)

        return relationship_id

    def remove_relationship(self, relationship_id: str) -> None:
        """
        Remove a relationship from the collection type.

        Args:
            relationship_id (str): The unique identifier of the relationship to remove.
        """
        if not relationship_id:
            raise ValueError(
                "You must provide a `relationship_id` to remove the relationship from the collection type."
            )
        if relationship_id not in self.relationships.keys():
            raise ValueError(
                f"Relationship with ID '{relationship_id}' does not exist in the collection type."
            )
        del self.relationships[relationship_id]

base_name property

Returns the entity name of the class as a string.

add(object_type, destination=None)

Add an object type to the collection type.

Parameters:

Name Type Description Default
object_type ObjectType

The object type to add to the collection type.

required
destination Destination | None

Optional openBIS destination override for this object. If omitted, the default destination configured in RunParsers is used.

None

Returns:

Name Type Description
str str

The unique identifier of the object type assigned in openBIS.

Source code in bam_masterdata/metadata/entities.py
def add(
    self,
    object_type: ObjectType,
    destination: Destination | None = None,
) -> str:
    """
    Add an object type to the collection type.

    Args:
        object_type (ObjectType): The object type to add to the collection type.
        destination (Destination | None): Optional openBIS destination override for this object.
            If omitted, the default destination configured in RunParsers is used.

    Returns:
        str: The unique identifier of the object type assigned in openBIS.
    """
    if not isinstance(object_type, ObjectType):
        raise TypeError(
            f"Expected an ObjectType instance, got `{type(object_type).__name__}`"
        )

    # Check mandatory properties are filled
    missing_fields = []
    for attr_name, prop in object_type._property_metadata.items():
        assigned_prop = getattr(object_type, attr_name, None)
        if prop.mandatory and isinstance(assigned_prop, PropertyTypeAssignment):
            missing_fields.append(attr_name)

    if missing_fields:
        raise ValueError(
            f"The following mandatory fields are missing for ObjectType '{object_type.cls_name}': {', '.join(missing_fields)}"
        )

    object_id = generate_object_id(object_type)
    self.attached_objects[object_id] = object_type
    if destination is not None:
        self.destinations[object_id] = destination
    return object_id

add_relationship(parent, child)

Add a relationship between two object types in the collection type.

Parameters:

Name Type Description Default
parent str | dict

Either the unique identifier of the parent object type or a dictionary representation of the parent object type.

required
child str | dict

Either the unique identifier of the child object type or a dictionary representation of the child object type.

required

Returns:

Name Type Description
str str

The unique identifier of the created relationship, generated from the resolved parent and child identifiers.

Source code in bam_masterdata/metadata/entities.py
def add_relationship(self, parent: str | dict, child: str | dict) -> str:
    """
    Add a relationship between two object types in the collection type.

    Args:
        parent (str | dict):
            Either the unique identifier of the parent object type or a
            dictionary representation of the parent object type.
        child (str | dict):
            Either the unique identifier of the child object type or a
            dictionary representation of the child object type.

    Returns:
        str:
            The unique identifier of the created relationship, generated
            from the resolved parent and child identifiers.
    """
    if not parent or not child:
        raise ValueError(
            "Both `parent` and `child` must be provided to add a relationship."
        )

    for obj in (parent, child):
        if not isinstance(obj, dict) and obj not in self.attached_objects:
            raise ValueError(
                "Both `parent` and `child` must be assigned to objects attached to the collection or being a dict readable by the system."
            )

    if not isinstance(parent, dict) and not isinstance(child, dict):
        relationship_id = generate_object_relationship_id(parent, child)
        self.relationships[relationship_id] = (parent, child)
    else:
        resolved_parent = (
            generate_dict_id(parent) if isinstance(parent, dict) else parent
        )

        resolved_child = (
            generate_dict_id(child) if isinstance(child, dict) else child
        )

        relationship_id = generate_object_relationship_id(
            resolved_parent, resolved_child
        )
        self.relationships[relationship_id] = (parent, child)

    return relationship_id

remove(object_id='')

Remove an object type from the collection type by its unique identifier.

Parameters:

Name Type Description Default
object_id str

The ID of the object type to be removed from the collection.

''
Source code in bam_masterdata/metadata/entities.py
def remove(self, object_id: str = "") -> None:
    """
    Remove an object type from the collection type by its unique identifier.

    Args:
        object_id (str, optional): The ID of the object type to be removed from the collection.
    """
    if not object_id:
        raise ValueError(
            "You must provide an `object_id` to remove the object type from the collection."
        )
    if object_id not in self.attached_objects.keys():
        raise ValueError(
            f"Object with ID '{object_id}' does not exist in the collection."
        )
    del self.attached_objects[object_id]
    self.destinations.pop(object_id, None)

remove_relationship(relationship_id)

Remove a relationship from the collection type.

Parameters:

Name Type Description Default
relationship_id str

The unique identifier of the relationship to remove.

required
Source code in bam_masterdata/metadata/entities.py
def remove_relationship(self, relationship_id: str) -> None:
    """
    Remove a relationship from the collection type.

    Args:
        relationship_id (str): The unique identifier of the relationship to remove.
    """
    if not relationship_id:
        raise ValueError(
            "You must provide a `relationship_id` to remove the relationship from the collection type."
        )
    if relationship_id not in self.relationships.keys():
        raise ValueError(
            f"Relationship with ID '{relationship_id}' does not exist in the collection type."
        )
    del self.relationships[relationship_id]

DatasetType

Bases: ObjectType

Source code in bam_masterdata/metadata/entities.py
class DatasetType(ObjectType):
    @property
    def base_name(self) -> str:
        """
        Returns the entity name of the class as a string.
        """
        return "DatasetType"

    @deprecated(
        "This method is deprecated and will be removed in a future major release. The creation of Object Types, "
        "Property Types, and Vocabularies is done now in the corresponding ObjectType and VocabularyType classes."
    )
    def to_openbis(
        self,
        logger: "BoundLoggerLazyProxy",
        openbis: "Openbis",
        type: str = "dataset",
        type_map: dict = DATASET_TYPE_MAP,
    ) -> None:
        def get_type(openbis: "Openbis", code: str):
            return openbis.get_dataset_type(code)

        def create_type(openbis: "Openbis", defs: DatasetTypeDef):
            return openbis.new_dataset_type(
                code=defs.code,
                description=defs.description,
                validationPlugin=defs.validation_script,
                # This is not accepted by openBIS when creating dataset types
                # mainDatasetPattern=defs.main_dataset_pattern,
                # mainDatasetPath=defs.main_dataset_path,
            )

        super()._to_openbis(
            logger=logger,
            openbis=openbis,
            type=type,
            type_map=type_map,
            get_type=get_type,
            create_type=create_type,
        )

base_name property

Returns the entity name of the class as a string.

VocabularyType

Bases: BaseEntity

Base class used to define vocabulary types. All vocabulary types must inherit from this class. The vocabulary types are defined in the module bam_masterdata/vocabulary_types.py.

The VocabularyType class contains a list of all terms defined for a VocabularyType, for internally represent the model in other formats (e.g., JSON or Excel).

Source code in bam_masterdata/metadata/entities.py
class VocabularyType(BaseEntity):
    """
    Base class used to define vocabulary types. All vocabulary types must inherit from this class. The
    vocabulary types are defined in the module `bam_masterdata/vocabulary_types.py`.

    The `VocabularyType` class contains a list of all `terms` defined for a `VocabularyType`, for
    internally represent the model in other formats (e.g., JSON or Excel).
    """

    model_config = ConfigDict(ignored_types=(VocabularyTypeDef, VocabularyTerm))

    terms: list[VocabularyTerm] = Field(
        default_factory=list,
        description="""
        List of vocabulary terms. This is useful for internal representation of the model.
        """,
    )

    @property
    def base_name(self) -> str:
        """
        Returns the entity name of the class as a string.
        """
        return "VocabularyType"

    @model_validator(mode="after")
    @classmethod
    def model_validator_after_init(cls, data: Any) -> Any:
        """
        Validate the model after instantiation of the class.

        Args:
            data (Any): The data containing the fields values to validate.

        Returns:
            Any: The data with the validated fields.
        """
        # Add all the vocabulary terms defined in the vocabulary type to the `terms` list.
        for base in cls.__mro__:
            for _, attr_val in base.__dict__.items():
                if isinstance(attr_val, VocabularyTerm):
                    data.terms.append(attr_val)

        return data

    @deprecated(
        "This method will soon be deprecated and will be removed in a future major release. "
        "The push to openbis and synchronization happens in `to_openbis_sync()` instead."
    )
    def to_openbis(self, logger: "BoundLoggerLazyProxy", openbis: "Openbis") -> None:
        openbis_entities = OpenbisEntities(url=openbis.url).get_vocabulary_dict()

        if self.defs.code in openbis_entities:
            logger.info(f"Vocabulary '{self.defs.code}' already exists in openBIS.")
            entity = openbis.get_vocabulary(self.defs.code)

            obis_terms = entity.get_terms().df.code
            obis_term_codes = [prop for prop in obis_terms]

            # Check for properties in self that are not in openBIS
            new_terms_added = False
            for term in self.terms:
                if term.code not in obis_term_codes:
                    logger.info(
                        f"Adding new term '{term.code}' to vocabulary '{self.defs.code}'."
                    )
                    new_terms_added = True

                    # Assign the term to the vocabulary
                    new_term = openbis.new_term(
                        code=term.code,
                        vocabularyCode=self.defs.code,
                        label=term.label,
                        description=term.description,
                    )
                    if term.official:
                        new_term.official = term.official
                    new_term.save()

            if not new_terms_added:
                logger.info(f"No new terms added to vocabulary '{self.defs.code}'.")

        else:
            logger.info(f"Creating new vocabulary '{self.defs.code}' in openBIS.")

            obis_terms = []
            obis_term_codes = []
            for term in self.terms:
                obis_terms.append(
                    {
                        "code": term.code,
                        "label": term.label,
                        "description": term.description,
                    }
                )
                obis_term_codes.append(term.code)
            logger.info(f"Adding terms {obis_term_codes} to {self.defs.code}.")

            # Creating the vocabulary in openBIS with its terms
            entity = openbis.new_vocabulary(
                code=self.defs.code,
                description=self.defs.description,
                terms=obis_terms,
            )
        entity.save()
        return entity

    def _create_vocabulary(
        self,
        openbis: "Openbis",
        result: SyncResult,
    ) -> None:
        """
        Creates the vocabulary and all terms defined in the model.

        Args:
            openbis (Openbis): The openBIS instance to synchronize this vocabulary to.
            result (SyncResult): Metadata describing the synchronization operations.
        """
        # Append all terms to the vocabulary in the format required by openBIS
        obis_terms = []
        for term in self.terms:
            obis_terms.append(
                {
                    "code": term.code,
                    "label": term.label,
                    "description": term.description,
                    # 'official': term.official,
                }
            )

        vocabulary = openbis.new_vocabulary(
            code=self.defs.code,
            description=self.defs.description,
            terms=obis_terms,
        )
        vocabulary.save()

        result.created(
            target=SyncTarget.VOCABULARY,
            code=self.defs.code,
        )

    def _sync_vocabulary_definition(
        self,
        vocabulary: "Vocabulary",
        result: SyncResult,
    ) -> None:
        """
        Synchronizes the fields of an existing vocabulary that are allowed
        to be modified.

        Currently only the description can be modified.
        """
        if vocabulary.description != self.defs.description:
            old_value = vocabulary.description
            vocabulary.description = self.defs.description

            result.modified(
                target=SyncTarget.VOCABULARY,
                code=self.defs.code,
                field="description",
                old_value=old_value,
                new_value=self.defs.description,
            )

            vocabulary.save()

    def _update_existing_vocabulary_term(
        self,
        term: VocabularyTerm,
        openbis_term,
        result: SyncResult,
    ) -> None:
        """
        Synchronizes an existing vocabulary term.

        Supported modifications:
        - label
        - description

        All other term metadata is immutable.
        """
        modified = False

        # Supported: label
        if openbis_term.label != term.label:
            old_value = openbis_term.label
            openbis_term.label = term.label
            modified = True

            result.modified(
                target=SyncTarget.VOCABULARY_TERM,
                code=term.code,
                parent_code=self.defs.code,
                field="label",
                old_value=old_value,
                new_value=term.label,
            )

        # Supported: description
        if openbis_term.description != term.description:
            old_value = openbis_term.description
            openbis_term.description = term.description
            modified = True

            result.modified(
                target=SyncTarget.VOCABULARY_TERM,
                code=term.code,
                parent_code=self.defs.code,
                field="description",
                old_value=old_value,
                new_value=term.description,
            )

        # Forbidden: official
        current_official = getattr(openbis_term, "official", False)
        if current_official != term.official:
            result.rejected(
                target=SyncTarget.VOCABULARY_TERM,
                code=term.code,
                parent_code=self.defs.code,
                field="official",
                old_value=current_official,
                new_value=term.official,
            )

        if modified:
            openbis_term.save()

    def _sync_vocabulary_terms(
        self,
        vocabulary: "Vocabulary",
        openbis: "Openbis",
        result: SyncResult,
    ) -> None:
        """
        Synchronizes the vocabulary terms.

        Supported operations:
        - add new terms
        - modify label
        - modify description

        Deleting existing terms is explicitly forbidden.
        """
        openbis_terms = vocabulary.get_terms()
        openbis_terms_by_code = openbis_terms.df["code"]
        openbis_codes = set(openbis_terms_by_code)

        model_terms = {term.code: term for term in self.terms}
        model_codes = set(model_terms)

        new_codes = model_codes - openbis_codes
        existing_codes = model_codes & openbis_codes
        missing_codes = openbis_codes - model_codes

        # Add new terms
        for code in new_codes:
            term = openbis.new_term(
                code=model_terms[code].code,
                vocabularyCode=self.defs.code,
                label=model_terms[code].label,
                description=model_terms[code].description,
            )
            term.save()
            result.created(
                target=SyncTarget.VOCABULARY_TERM,
                code=code,
                parent_code=self.defs.code,
            )

        # Synchronize allowed fields of existing terms
        for code in existing_codes:
            self._update_existing_vocabulary_term(
                term=model_terms[code],
                openbis_term=openbis_terms[code],
                result=result,
            )

        # Terms existing in openBIS but missing from the model would imply deletion.
        # Deletion is explicitly forbidden.
        for code in missing_codes:
            result.rejected(
                target=SyncTarget.VOCABULARY_TERM,
                code=code,
                parent_code=self.defs.code,
                field="code",
                old_value=code,
                new_value=None,
                message="Deletion of vocabulary terms is not allowed.",
            )

    def to_openbis_sync(self, openbis: "Openbis") -> SyncResult:
        """
        Synchronizes this VocabularyType definition and its VocabularyTerms
        with the openBIS instance.

        Supported operations:
        - create new vocabularies
        - modify vocabulary descriptions
        - add new vocabulary terms
        - modify vocabulary term labels
        - modify vocabulary term descriptions

        Forbidden operations:
        - delete vocabularies
        - delete vocabulary terms
        - modify vocabulary codes
        - modify vocabulary term codes
        - modify other vocabulary term metadata, such as `official`

        Args:
            openbis (Openbis):The openBIS instance to synchronize this vocabulary to.

        Returns:
            SyncResult: Metadata describing the synchronization operations.
        """
        result = SyncResult()

        openbis_vocabularies = OpenbisEntities(url=openbis).get_vocabulary_dict()

        # Create vocabulary and terms if missing
        if self.defs.code not in openbis_vocabularies:
            self._create_vocabulary(
                openbis=openbis,
                result=result,
            )
            return result

        vocabulary = openbis.get_vocabulary(self.defs.code)

        # Synchronize vocabulary metadata
        self._sync_vocabulary_definition(
            vocabulary=vocabulary,
            result=result,
        )

        # Synchronize terms
        self._sync_vocabulary_terms(
            vocabulary=vocabulary,
            openbis=openbis,
            result=result,
        )

        return result

base_name property

Returns the entity name of the class as a string.

model_validator_after_init(data) classmethod

Validate the model after instantiation of the class.

Parameters:

Name Type Description Default
data Any

The data containing the fields values to validate.

required

Returns:

Name Type Description
Any Any

The data with the validated fields.

Source code in bam_masterdata/metadata/entities.py
@model_validator(mode="after")
@classmethod
def model_validator_after_init(cls, data: Any) -> Any:
    """
    Validate the model after instantiation of the class.

    Args:
        data (Any): The data containing the fields values to validate.

    Returns:
        Any: The data with the validated fields.
    """
    # Add all the vocabulary terms defined in the vocabulary type to the `terms` list.
    for base in cls.__mro__:
        for _, attr_val in base.__dict__.items():
            if isinstance(attr_val, VocabularyTerm):
                data.terms.append(attr_val)

    return data

to_openbis_sync(openbis)

Synchronizes this VocabularyType definition and its VocabularyTerms with the openBIS instance.

Supported operations: - create new vocabularies - modify vocabulary descriptions - add new vocabulary terms - modify vocabulary term labels - modify vocabulary term descriptions

Forbidden operations: - delete vocabularies - delete vocabulary terms - modify vocabulary codes - modify vocabulary term codes - modify other vocabulary term metadata, such as official

Parameters:

Name Type Description Default
openbis Openbis

The openBIS instance to synchronize this vocabulary to.

required

Returns:

Name Type Description
SyncResult SyncResult

Metadata describing the synchronization operations.

Source code in bam_masterdata/metadata/entities.py
def to_openbis_sync(self, openbis: "Openbis") -> SyncResult:
    """
    Synchronizes this VocabularyType definition and its VocabularyTerms
    with the openBIS instance.

    Supported operations:
    - create new vocabularies
    - modify vocabulary descriptions
    - add new vocabulary terms
    - modify vocabulary term labels
    - modify vocabulary term descriptions

    Forbidden operations:
    - delete vocabularies
    - delete vocabulary terms
    - modify vocabulary codes
    - modify vocabulary term codes
    - modify other vocabulary term metadata, such as `official`

    Args:
        openbis (Openbis):The openBIS instance to synchronize this vocabulary to.

    Returns:
        SyncResult: Metadata describing the synchronization operations.
    """
    result = SyncResult()

    openbis_vocabularies = OpenbisEntities(url=openbis).get_vocabulary_dict()

    # Create vocabulary and terms if missing
    if self.defs.code not in openbis_vocabularies:
        self._create_vocabulary(
            openbis=openbis,
            result=result,
        )
        return result

    vocabulary = openbis.get_vocabulary(self.defs.code)

    # Synchronize vocabulary metadata
    self._sync_vocabulary_definition(
        vocabulary=vocabulary,
        result=result,
    )

    # Synchronize terms
    self._sync_vocabulary_terms(
        vocabulary=vocabulary,
        openbis=openbis,
        result=result,
    )

    return result

bam_masterdata.metadata.definitions

EntityDef

Bases: BaseModel

Abstract base class for all masterdata entity definitions. The entity definitions are immutable properties. This class provides a common interface (with common attributes like code and description.) for all entity definitions.

Source code in bam_masterdata/metadata/definitions.py
class EntityDef(BaseModel):
    """
    Abstract base class for all masterdata entity definitions. The entity definitions are immutable properties.
    This class provides a common interface (with common attributes like `code` and
    `description`.) for all entity definitions.
    """

    code: str = Field(
        ...,
        description="""
        Code string identifying the entity with an openBIS inventory definition. Note that:

        - Must be uppercase and separated by underscores, e.g. `'EXPERIMENTAL_STEP'`.
        - If the entity is native to openBIS, the code must start with a dollar sign, e.g. `'$NAME'`.
        - In the case of inheritance, it needs to be separated by dots, e.g. `'WELDING_EQUIPMENT.INSTRUMENT'`.
        """,
    )

    description: str = Field(
        ...,
        description="""
        Description of the entity. This is the human-readable text for the object and must be
        as complete and concise as possible. The German description can be added after the English
        description separated by a double slash (//), e.g. `'Chemical Substance//Chemische Substanz'`.
        """,
    )

    # TODO: check if it is necessary to add something like `ontology_annotation_id` in the future
    iri: str | None = Field(
        default=None,
        description="""
        IRI (Internationalized Resource Identifier) of the entity. This is a unique identifier for the entity
        and is used to link the entity to an ontology. It is a string with the format `"<ontology_id>:<ontology_version>"`.
        Example: "http://purl.obolibrary.org/bam-masterdata/Instrument:1.0.0".
        """,
    )

    id: str | None = Field(
        default=None,
        description="""
        Identifier of the entity defined as the class name and used to serialize the entity definitions
        in other formats.
        """,
    )

    row_location: str | None = Field(
        default=None,
        description="""
        Row in the Excel at which the entity type field is defined. It is a string with the format `"<row-letter><row_number>"`.
        Example: "A1" ot "A107". This field is useful when checking the consistency of Excel files with multiple entity
        types defined to quickly locate the specific Excel cell which logs a message when applying the `checker` CLI.
        """,
    )

    # TODO check ontology_id, ontology_version, ontology_annotation_id, internal (found in the openBIS docu)

    @field_validator("code")
    @classmethod
    def validate_code(cls, value: str) -> str:
        """
        Validate entity codes against the allowed openBIS format.

        Args:
            value: Candidate code string.

        Returns:
            The validated code.

        Raises:
            ValueError: If the code is empty or contains invalid characters.
        """
        if not value or not re.match(r"^[\w_\$\.\-\+]+$", value):
            raise ValueError(
                "`code` must follow the rules specified in the description: 1) Must be uppercase, "
                "2) separated by underscores, 3) start with a dollar sign if native to openBIS, "
                "4) separated by dots if there is inheritance."
            )
        return value

    @field_validator("iri")
    @classmethod
    def validate_iri(cls, value: str | None) -> str | None:
        """
        Validate the optional ontology IRI against the expected BAM pattern.

        Args:
            value: Candidate IRI string, or None.

        Returns:
            The validated IRI (or None).

        Raises:
            ValueError: If the IRI does not match the required pattern.
        """
        if not value:
            return value
        if not re.match(
            r"^http://purl.obolibrary.org/bam-masterdata/[\w_]+:[\d.]+$", value
        ):
            raise ValueError(
                "`iri` must follow the rules specified in the description: 1) Must start with 'http://purl.obolibrary.org/bam-masterdata/', "
                "2) followed by the entity name, 3) separated by a colon, 4) followed by the semantic versioning number. "
                "Example: 'http://purl.obolibrary.org/bam-masterdata/Instrument:1.0.0'."
            )
        return value

    @field_validator("description")
    @classmethod
    def strip_description(cls, value: str) -> str:
        """
        Strip leading and trailing whitespace from descriptions.

        Args:
            value: Description string.

        Returns:
            The trimmed description.
        """
        return value.strip()

    @property
    def name(self) -> str:
        return self.__class__.__name__

    @property
    def excel_name(self) -> str:
        """
        Returns the name of the entity in a format suitable for the openBIS Excel file.
        """
        return _EXCEL_NAME_MAP.get(self.name)

    @property
    def excel_headers_map(self) -> dict:
        """
        Maps the field keys of the Pydantic model into the openBIS Excel style headers.
        """
        return {
            field_name: _format_excel_header(field_name)
            for field_name in self.model_fields.keys()
            if field_name not in _EXCLUDED_EXCEL_HEADER_FIELDS
        }

    @model_validator(mode="after")
    @classmethod
    def model_id(cls, data: Any) -> Any:
        """
        Populate `id` based on the entity code and entity type.

        Args:
            data (Any): The data containing the fields values to validate.

        Returns:
            Any: The data with the validated fields.
        """
        data.id = code_to_class_name(
            code=data.code,
            entity_type=_entity_type_for_model_id(data.name),
        )
        return data

excel_headers_map property

Maps the field keys of the Pydantic model into the openBIS Excel style headers.

excel_name property

Returns the name of the entity in a format suitable for the openBIS Excel file.

model_id(data) classmethod

Populate id based on the entity code and entity type.

Parameters:

Name Type Description Default
data Any

The data containing the fields values to validate.

required

Returns:

Name Type Description
Any Any

The data with the validated fields.

Source code in bam_masterdata/metadata/definitions.py
@model_validator(mode="after")
@classmethod
def model_id(cls, data: Any) -> Any:
    """
    Populate `id` based on the entity code and entity type.

    Args:
        data (Any): The data containing the fields values to validate.

    Returns:
        Any: The data with the validated fields.
    """
    data.id = code_to_class_name(
        code=data.code,
        entity_type=_entity_type_for_model_id(data.name),
    )
    return data

strip_description(value) classmethod

Strip leading and trailing whitespace from descriptions.

Parameters:

Name Type Description Default
value str

Description string.

required

Returns:

Type Description
str

The trimmed description.

Source code in bam_masterdata/metadata/definitions.py
@field_validator("description")
@classmethod
def strip_description(cls, value: str) -> str:
    """
    Strip leading and trailing whitespace from descriptions.

    Args:
        value: Description string.

    Returns:
        The trimmed description.
    """
    return value.strip()

validate_code(value) classmethod

Validate entity codes against the allowed openBIS format.

Parameters:

Name Type Description Default
value str

Candidate code string.

required

Returns:

Type Description
str

The validated code.

Raises:

Type Description
ValueError

If the code is empty or contains invalid characters.

Source code in bam_masterdata/metadata/definitions.py
@field_validator("code")
@classmethod
def validate_code(cls, value: str) -> str:
    """
    Validate entity codes against the allowed openBIS format.

    Args:
        value: Candidate code string.

    Returns:
        The validated code.

    Raises:
        ValueError: If the code is empty or contains invalid characters.
    """
    if not value or not re.match(r"^[\w_\$\.\-\+]+$", value):
        raise ValueError(
            "`code` must follow the rules specified in the description: 1) Must be uppercase, "
            "2) separated by underscores, 3) start with a dollar sign if native to openBIS, "
            "4) separated by dots if there is inheritance."
        )
    return value

validate_iri(value) classmethod

Validate the optional ontology IRI against the expected BAM pattern.

Parameters:

Name Type Description Default
value str | None

Candidate IRI string, or None.

required

Returns:

Type Description
str | None

The validated IRI (or None).

Raises:

Type Description
ValueError

If the IRI does not match the required pattern.

Source code in bam_masterdata/metadata/definitions.py
@field_validator("iri")
@classmethod
def validate_iri(cls, value: str | None) -> str | None:
    """
    Validate the optional ontology IRI against the expected BAM pattern.

    Args:
        value: Candidate IRI string, or None.

    Returns:
        The validated IRI (or None).

    Raises:
        ValueError: If the IRI does not match the required pattern.
    """
    if not value:
        return value
    if not re.match(
        r"^http://purl.obolibrary.org/bam-masterdata/[\w_]+:[\d.]+$", value
    ):
        raise ValueError(
            "`iri` must follow the rules specified in the description: 1) Must start with 'http://purl.obolibrary.org/bam-masterdata/', "
            "2) followed by the entity name, 3) separated by a colon, 4) followed by the semantic versioning number. "
            "Example: 'http://purl.obolibrary.org/bam-masterdata/Instrument:1.0.0'."
        )
    return value

ObjectTypeDef

Bases: BaseObjectTypeDef

Definition class for an object type. It adds the fields of generated_code_prefix, auto_generate_codes, and validation_script to the common attributes of a base object type definition. E.g.:

class Instrument(BaseModel):
    defs = ObjectTypeDef(
        code='INSTRUMENT',
        description='
        Measuring Instrument//Messgerät
        ',
        generated_code_prefix='INS',
    )
Source code in bam_masterdata/metadata/definitions.py
class ObjectTypeDef(BaseObjectTypeDef):
    """
    Definition class for an object type. It adds the fields of `generated_code_prefix`, `auto_generate_codes`,
    and `validation_script` to the common attributes of a base object type definition. E.g.:

    ```python
    class Instrument(BaseModel):
        defs = ObjectTypeDef(
            code='INSTRUMENT',
            description='
            Measuring Instrument//Messger\u00e4t
            ',
            generated_code_prefix='INS',
        )
    ```
    """

    generated_code_prefix: str | None = Field(
        default=None,
        description="""
        A short prefix for the defined object type, e.g. 'CHEM'. If not specified, it is defined
        using the first 3 characters of `code`.
        """,
    )

    auto_generate_codes: bool = Field(
        True,
        description="""
        Boolean used to generate codes using `generated_code_prefix` plus a unique number. Set to
        True by default.
        """,
    )

    @model_validator(mode="after")
    @classmethod
    def model_validator_after_init(cls, data: Any) -> Any:
        """
        Ensure `generated_code_prefix` is set after initialization.

        Args:
            data (Any): The data containing the fields values to validate.

        Returns:
            Any: The data with the validated fields.
        """
        # If `generated_code_prefix` is not set, use the first 3 characters of `code`
        if not data.generated_code_prefix:
            data.generated_code_prefix = data.code[:3]

        return data

model_validator_after_init(data) classmethod

Ensure generated_code_prefix is set after initialization.

Parameters:

Name Type Description Default
data Any

The data containing the fields values to validate.

required

Returns:

Name Type Description
Any Any

The data with the validated fields.

Source code in bam_masterdata/metadata/definitions.py
@model_validator(mode="after")
@classmethod
def model_validator_after_init(cls, data: Any) -> Any:
    """
    Ensure `generated_code_prefix` is set after initialization.

    Args:
        data (Any): The data containing the fields values to validate.

    Returns:
        Any: The data with the validated fields.
    """
    # If `generated_code_prefix` is not set, use the first 3 characters of `code`
    if not data.generated_code_prefix:
        data.generated_code_prefix = data.code[:3]

    return data

CollectionTypeDef

Bases: BaseObjectTypeDef

Definition class for a collection type. E.g.:

class DefaultExperiment(BaseModel):
    defs = CollectionTypeDef(
        code='DEFAULT_EXPERIMENT',
        description='...',
        validation_script='DEFAULT_EXPERIMENT.date_range_validation',
    )
Source code in bam_masterdata/metadata/definitions.py
class CollectionTypeDef(BaseObjectTypeDef):
    """
    Definition class for a collection type. E.g.:

    ```python
    class DefaultExperiment(BaseModel):
        defs = CollectionTypeDef(
            code='DEFAULT_EXPERIMENT',
            description='...',
            validation_script='DEFAULT_EXPERIMENT.date_range_validation',
        )
    ```
    """

    pass

DatasetTypeDef

Bases: BaseObjectTypeDef

Definition class for a data set type. E.g.:

```python class RawData(BaseModel): defs = DatasetTypeDef( code='RAW_DATA', description='...', )

Source code in bam_masterdata/metadata/definitions.py
class DatasetTypeDef(BaseObjectTypeDef):
    """
    Definition class for a data set type. E.g.:

    ```python
    class RawData(BaseModel):
        defs = DatasetTypeDef(
            code='RAW_DATA',
            description='...',
        )
    """

    # TODO add descriptions for `main_dataset_pattern` and `main_dataset_path`

    main_dataset_pattern: str | None = Field(
        default=None,
        description="""""",
    )

    main_dataset_path: str | None = Field(
        default=None,
        description="""""",
    )

VocabularyTypeDef

Bases: EntityDef

Definition class for a vocabulary type. It adds the fields of url_template to the common attributes of an entity definition. E.g.:

class DocumentType(VocabularyType):
    defs = VocabularyTypeDef(
        code='DOCUMENT_TYPE',
        description='Document type//Dokumententypen',
    )
Source code in bam_masterdata/metadata/definitions.py
class VocabularyTypeDef(EntityDef):
    """
    Definition class for a vocabulary type. It adds the fields of `url_template` to the common attributes of
    an entity definition. E.g.:

    ```python
    class DocumentType(VocabularyType):
        defs = VocabularyTypeDef(
            code='DOCUMENT_TYPE',
            description='Document type//Dokumententypen',
        )
    ```
    """

    # TODO add descriptions for `url_template`

    url_template: str | None = Field(
        default=None,
        description="""""",
    )

PropertyTypeDef

Bases: EntityDef

Definition class for a property type. It adds the fields of property_label, data_type, vocabulary_code, metadata, dynamic_script, and multivalued to the common attributes of an entity definition.

This class is used as an abstract layer for PropertyTypeAssignment, as in openBIS a PropertyType definition has less fields than when it is actually assigned to an entity type.

Source code in bam_masterdata/metadata/definitions.py
class PropertyTypeDef(EntityDef):
    """
    Definition class for a property type. It adds the fields of `property_label`, `data_type`,
    `vocabulary_code`, `metadata`, `dynamic_script`, and `multivalued` to the common attributes of
    an entity definition.

    This class is used as an abstract layer for `PropertyTypeAssignment`, as in openBIS a PropertyType
    definition has less fields than when it is actually assigned to an entity type.
    """

    property_label: str = Field(
        ...,
        description="""
        Label that appears in the inventory view. This is the human-readable text for the property
        type definition, and it typically coincides with the `code`, e.g., `'Monitoring date'` for the
        `MONITORING_DATE` property type.
        """,
    )

    units: str | None = Field(
        default=None,
        description="""
        Optional units for the property type, expressed in pint format. Read more about pint
        units in https://github.com/hgrecco/pint/blob/master/pint/default_en.txt.

        When provided, the `property_label` is automatically suffixed with `in [units]` unless
        it already contains a units suffix in square brackets (legacy feature).
        """,
    )

    data_type: DataType = Field(
        ...,
        description="""
        The data type of the property, i.e., if it is an integer, float, string, etc. The allowed
        data types in openBIS are:
            - `BOOLEAN`
            - `CONTROLLEDVOCABULARY`
            - `DATE`
            - `HYPERLINK`
            - `INTEGER`
            - `MATERIAL`
            - `MULTILINE_VARCHAR`
            - `OBJECT`
            - `SAMPLE`
            - `REAL`
            - `TIMESTAMP`
            - `VARCHAR`
            - `XML`

        These are defined as an enumeration in the `DataType` class.

        Read more in https://openbis.readthedocs.io/en/latest/uncategorized/register-master-data-via-the-admin-interface.html#data-types-available-in-openbis.
        """,
    )

    vocabulary_code: str | None = Field(
        default=None,
        description="""
        String identifying the controlled vocabulary used for the data type of the property. This is
        thus only relevant if `data_type == 'CONTROLLEDVOCABULARY'`.
        """,
    )

    object_code: str | None = Field(
        default=None,
        description="""
        String identifying the object type used for the data type of the property. This is only
        relevant if `data_type == 'OBJECT'`.
        """,
    )

    # TODO add descriptions for `dynamic_script`

    metadata: dict | None = Field(
        default=None,
        description="""
        General metadata written in a dictionary format. This is used to store additional information
        about the property type, e.g., `{'unit': 'm', 'precision': 2}`.
        """,
    )

    dynamic_script: str | None = Field(
        default=None,
        description="""""",
    )

    ordinal: int | None = Field(
        default=None,
        description="""
        Ordinal of the property type. This is used to order the properties in the inventory view
        """,
    )

    @field_validator("units")
    @classmethod
    def validate_units(cls, value: str | None) -> str | None:
        """
        Validate the optional units string using pint's unit registry.

        Args:
            value: Units string in pint format, or None.

        Returns:
            The validated units string (or None).

        Raises:
            ValueError: If the units string is not recognized by pint.
        """
        if value is None:
            return value
        try:
            _UNIT_REGISTRY.Unit(value)
        except (UndefinedUnitError, DefinitionSyntaxError) as exc:
            raise ValueError(f"Invalid units format: {value}") from exc
        return value

    @model_validator(mode="after")
    @classmethod
    def apply_units_to_property_label(cls, data: Any) -> Any:
        """
        Enforce the `in [units]` suffix in `property_label` when units are provided.

        If the label already contains a bracketed unit, it must include the
        exact `in [units]` format or validation fails. Otherwise, the suffix
        is appended.

        Args:
            data (Any): The data containing the fields values to validate.

        Returns:
            Any: The data with the updated label (if needed).

        Raises:
            ValueError: If a bracketed unit exists without the `in [units]` suffix.
        """
        if data.units:
            if _UNIT_LABEL_PATTERN.search(data.property_label):
                if not _UNIT_SUFFIX_PATTERN.search(data.property_label):
                    raise ValueError(
                        "property_label with units must include 'in [units]'"
                    )
            else:
                data.property_label += f" in [{data.units}]"
        return data

apply_units_to_property_label(data) classmethod

Enforce the in [units] suffix in property_label when units are provided.

If the label already contains a bracketed unit, it must include the exact in [units] format or validation fails. Otherwise, the suffix is appended.

Parameters:

Name Type Description Default
data Any

The data containing the fields values to validate.

required

Returns:

Name Type Description
Any Any

The data with the updated label (if needed).

Raises:

Type Description
ValueError

If a bracketed unit exists without the in [units] suffix.

Source code in bam_masterdata/metadata/definitions.py
@model_validator(mode="after")
@classmethod
def apply_units_to_property_label(cls, data: Any) -> Any:
    """
    Enforce the `in [units]` suffix in `property_label` when units are provided.

    If the label already contains a bracketed unit, it must include the
    exact `in [units]` format or validation fails. Otherwise, the suffix
    is appended.

    Args:
        data (Any): The data containing the fields values to validate.

    Returns:
        Any: The data with the updated label (if needed).

    Raises:
        ValueError: If a bracketed unit exists without the `in [units]` suffix.
    """
    if data.units:
        if _UNIT_LABEL_PATTERN.search(data.property_label):
            if not _UNIT_SUFFIX_PATTERN.search(data.property_label):
                raise ValueError(
                    "property_label with units must include 'in [units]'"
                )
        else:
            data.property_label += f" in [{data.units}]"
    return data

validate_units(value) classmethod

Validate the optional units string using pint's unit registry.

Parameters:

Name Type Description Default
value str | None

Units string in pint format, or None.

required

Returns:

Type Description
str | None

The validated units string (or None).

Raises:

Type Description
ValueError

If the units string is not recognized by pint.

Source code in bam_masterdata/metadata/definitions.py
@field_validator("units")
@classmethod
def validate_units(cls, value: str | None) -> str | None:
    """
    Validate the optional units string using pint's unit registry.

    Args:
        value: Units string in pint format, or None.

    Returns:
        The validated units string (or None).

    Raises:
        ValueError: If the units string is not recognized by pint.
    """
    if value is None:
        return value
    try:
        _UNIT_REGISTRY.Unit(value)
    except (UndefinedUnitError, DefinitionSyntaxError) as exc:
        raise ValueError(f"Invalid units format: {value}") from exc
    return value

PropertyTypeAssignment

Bases: PropertyTypeDef

Base class used to define properties inside ObjectType, CollectionType, or DatasetType. This is used to construct these types by assigning property types to them. It adds the fields of mandatory, show_in_edit_views, section, unique, and internal_assignment to the common attributes of a property type definition. E.g.:

class Instrument(ObjectType):
    defs = ObjectTypeDef(
        code='INSTRUMENT',
        description='
        Measuring Instrument//Messgerät
        ',
        generated_code_prefix='INS',
    )

    alias = PropertyTypeAssignment(
        code='ALIAS',
        data_type='VARCHAR',
        property_label='Alternative name',
        description='
        e.g. abbreviation or nickname//z.B. Abkürzung oder Spitzname//z.B. Abkürzung oder Spitzname
        ',
        mandatory=False,
        show_in_edit_views=True,
        section='General information',
    )

    # ... other property type assignments here ...
Source code in bam_masterdata/metadata/definitions.py
class PropertyTypeAssignment(PropertyTypeDef):
    """
    Base class used to define properties inside `ObjectType`, `CollectionType`, or `DatasetType`.
    This is used to construct these types by assigning property types to them. It adds the fields
    of `mandatory`, `show_in_edit_views`, `section`, `unique`, and `internal_assignment` to the common
    attributes of a property type definition. E.g.:

    ```python
    class Instrument(ObjectType):
        defs = ObjectTypeDef(
            code='INSTRUMENT',
            description='
            Measuring Instrument//Messger\u00e4t
            ',
            generated_code_prefix='INS',
        )

        alias = PropertyTypeAssignment(
            code='ALIAS',
            data_type='VARCHAR',
            property_label='Alternative name',
            description='
            e.g. abbreviation or nickname//z.B. Abkürzung oder Spitzname//z.B. Abkürzung oder Spitzname
            ',
            mandatory=False,
            show_in_edit_views=True,
            section='General information',
        )

        # ... other property type assignments here ...
    ```
    """

    mandatory: bool = Field(
        ...,
        description="""
        If `True`, the property is mandatory and has to be set during instantiation of the object type.
        If `False`, the property is optional.
        """,
    )

    show_in_edit_views: bool = Field(
        True,
        description="""
        If `True`, the property is shown in the edit views of the ELN in the object type instantiation.
        If `False`, the property is hidden.
        Set to `True` by default.
        """,
    )

    section: str = Field(
        ...,
        description="""
        Section to which the property type belongs to. E.g., `'General Information'`.
        """,
    )

    # TODO add descriptions for `unique` and `internal_assignment`

    unique: str | None = Field(
        default=None,
        description="""""",
    )

    internal_assignment: str | None = Field(
        default=None,
        description="""""",
    )

VocabularyTerm

Bases: VocabularyTypeDef

Base class used to define terms inside a VocabularyType. This is used to construct the vocabulary types by assigning vocabulary terms to them. It adds the fields of label and official to the common attributes of a vocabulary type definition. E.g.:

```python class DocumentType(VocabularyType): defs = VocabularyTypeDef( code='DOCUMENT_TYPE', description='Document type//Dokumententypen', )

acceptance_certificate = VocabularyTerm(
    code='ACCEPTANCE_CERTIFICATE',
    label='Acceptance Certificate',
    description='Acceptance Certificate//Abnahmezeugnis',
)

calibration_certificate = VocabularyTerm(
    code='CALIBRATION_CERTIFICATE',
    label='Calibration Certificate',
    description='Calibration Certificate//Kalibrierschein',
)

# ... other vocabulary term definitions here ...
Source code in bam_masterdata/metadata/definitions.py
class VocabularyTerm(VocabularyTypeDef):
    """
    Base class used to define terms inside a `VocabularyType`. This is used to construct the vocabulary types
    by assigning vocabulary terms to them. It adds the fields of `label` and `official` to the common attributes
    of a vocabulary type definition. E.g.:

    ```python
    class DocumentType(VocabularyType):
        defs = VocabularyTypeDef(
            code='DOCUMENT_TYPE',
            description='Document type//Dokumententypen',
        )

        acceptance_certificate = VocabularyTerm(
            code='ACCEPTANCE_CERTIFICATE',
            label='Acceptance Certificate',
            description='Acceptance Certificate//Abnahmezeugnis',
        )

        calibration_certificate = VocabularyTerm(
            code='CALIBRATION_CERTIFICATE',
            label='Calibration Certificate',
            description='Calibration Certificate//Kalibrierschein',
        )

        # ... other vocabulary term definitions here ...
    """

    # TODO add descriptions for `label` and `official`

    label: str = Field(
        ...,
        description="""""",
    )

    official: bool = Field(
        True,
        description="""""",
    )

    @field_validator("code")
    @classmethod
    def validate_code_length(cls, value: str) -> str:
        if len(value) > 50:
            raise ValueError(
                f"VocabularyTerm code must not exceed 50 characters (got {len(value)} characters)."
            )
        return value

bam_masterdata.metadata.entities_dict

EntitiesDict

Class to convert the entities in the datamodel defined in Python to a dictionary. The entities are read from the Python files defined in python_path.

Source code in bam_masterdata/metadata/entities_dict.py
class EntitiesDict:
    """
    Class to convert the entities in the datamodel defined in Python to a dictionary. The entities are read from the Python
    files defined in `python_path`.
    """

    def __init__(self, python_path: str = "", **kwargs):
        self.python_path = python_path
        self.logger = kwargs.get("logger", logger)
        self.data: dict = {}

    @staticmethod
    def _read_module_source(module_path: str) -> list[str]:
        with open(module_path, encoding="utf-8") as module_file:
            return module_file.readlines()

    @staticmethod
    def _collect_class_locations(module_source: list[str]) -> dict[str, int]:
        return {
            match.group(1): line_number
            for line_number, line in enumerate(module_source, start=1)
            if (match := CLASS_DEFINITION_PATTERN.match(line))
        }

    @staticmethod
    def _collect_member_locations(
        module_source: list[str], constructor_name: str
    ) -> dict[str, dict[str, int]]:
        member_pattern = re.compile(rf"^\s*(\w+)\s*=\s*{re.escape(constructor_name)}\(")
        member_locations: dict[str, dict[str, int]] = {}
        current_class = None

        for line_number, line in enumerate(module_source, start=1):
            class_match = CLASS_DEFINITION_PATTERN.match(line)
            if class_match:
                current_class = class_match.group(1)
                continue

            member_match = member_pattern.search(line)
            if member_match and current_class:
                member_name = member_match.group(1)
                member_locations.setdefault(current_class, {})[member_name] = (
                    line_number
                )

        return member_locations

    @staticmethod
    def _code_to_attribute_name(code: str, *, strip_dollar: bool = False) -> str:
        normalized_code = code.lower().replace(".", "_")
        if strip_dollar:
            normalized_code = normalized_code.replace("$", "")
        return normalized_code

    @classmethod
    def _assign_row_locations(
        cls,
        entity_name: str,
        items: list[dict],
        member_locations: dict[str, dict[str, int]],
        *,
        strip_dollar: bool = False,
    ) -> None:
        entity_locations = member_locations.get(entity_name, {})
        for item in items:
            attr_name = cls._code_to_attribute_name(
                item["code"], strip_dollar=strip_dollar
            )
            item["row_location"] = entity_locations.get(attr_name)

    def to_dict(self, module_path: str) -> dict:
        """
        Returns a dictionary containing entities read from the `module_path` Python file. The Python modules
        are imported using the function `import_module` and their contents are inspected (using `inspect`) to
        find the classes in the datamodel containing `defs` and with a `model_to_dict` method defined.

        Args:
            module_path (str): Path to the Python module file.

        Returns:
            dict: A dictionary containing the entities in the datamodel defined in one Python module file.
        """
        module = import_module(module_path=module_path)

        # initializing the dictionary with keys as the `code` of the entity and values the json dumped data
        data: dict = {}

        # Read the module source code and store line numbers
        module_source = self._read_module_source(module_path)
        class_locations = self._collect_class_locations(module_source)
        property_locations = self._collect_member_locations(
            module_source, "PropertyTypeAssignment"
        )
        vocabulary_term_locations = self._collect_member_locations(
            module_source, "VocabularyTerm"
        )

        # Process all classes in the module
        for name, obj in inspect.getmembers(module, inspect.isclass):
            if not hasattr(obj, "defs") or not callable(getattr(obj, "model_to_dict")):
                continue
            try:
                obj_data = obj().model_to_dict()
                obj_data["defs"]["row_location"] = class_locations.get(name, None)

                if "properties" in obj_data:
                    self._assign_row_locations(
                        name,
                        obj_data["properties"],
                        property_locations,
                        strip_dollar=True,
                    )

                elif "terms" in obj_data:
                    self._assign_row_locations(
                        name, obj_data["terms"], vocabulary_term_locations
                    )

                data[obj.defs.code] = obj_data
            except Exception as err:
                click.echo(f"Failed to process class {name} in {module_path}: {err}")

        return data

    def single_json(self) -> dict:
        """
        Returns a single dictionary containing all the entities in the datamodel defined in the Python files
        in `python_path`. The format of this dictionary is:
            {
                "collection_type": {
                    "COLLECTION": {
                        "defs": {
                            "code": "COLLECTION",
                            "description": "",
                            ...
                        },
                        "properties": [
                            {
                                "code": "$DEFAULT_COLLECTION_VIEW",
                                "description": "Default view for experiments of the type collection",
                                ...
                            },
                            {...},
                            ...
                        ]
                    }
                },
                "object_type": {...},
                ...
            }

        Returns:
            dict: A dictionary containing all the entities in the datamodel.
        """
        # Get the Python modules to process the datamodel
        py_modules = listdir_py_modules(
            directory_path=self.python_path, logger=self.logger
        )

        # Process each module using the `model_to_dict` method of each entity and store them in a single dictionary
        full_data: dict = {}
        for module_path in py_modules:
            data = self.to_dict(module_path=module_path)
            # name can be collection_type, object_type, dataset_type, vocabulary_type, or property_type
            name = os.path.basename(module_path).replace(".py", "")
            full_data[name] = data
        return full_data

single_json()

Returns a single dictionary containing all the entities in the datamodel defined in the Python files in python_path. The format of this dictionary is: { "collection_type": { "COLLECTION": { "defs": { "code": "COLLECTION", "description": "", ... }, "properties": [ { "code": "$DEFAULT_COLLECTION_VIEW", "description": "Default view for experiments of the type collection", ... }, {...}, ... ] } }, "object_type": {...}, ... }

Returns:

Name Type Description
dict dict

A dictionary containing all the entities in the datamodel.

Source code in bam_masterdata/metadata/entities_dict.py
def single_json(self) -> dict:
    """
    Returns a single dictionary containing all the entities in the datamodel defined in the Python files
    in `python_path`. The format of this dictionary is:
        {
            "collection_type": {
                "COLLECTION": {
                    "defs": {
                        "code": "COLLECTION",
                        "description": "",
                        ...
                    },
                    "properties": [
                        {
                            "code": "$DEFAULT_COLLECTION_VIEW",
                            "description": "Default view for experiments of the type collection",
                            ...
                        },
                        {...},
                        ...
                    ]
                }
            },
            "object_type": {...},
            ...
        }

    Returns:
        dict: A dictionary containing all the entities in the datamodel.
    """
    # Get the Python modules to process the datamodel
    py_modules = listdir_py_modules(
        directory_path=self.python_path, logger=self.logger
    )

    # Process each module using the `model_to_dict` method of each entity and store them in a single dictionary
    full_data: dict = {}
    for module_path in py_modules:
        data = self.to_dict(module_path=module_path)
        # name can be collection_type, object_type, dataset_type, vocabulary_type, or property_type
        name = os.path.basename(module_path).replace(".py", "")
        full_data[name] = data
    return full_data

to_dict(module_path)

Returns a dictionary containing entities read from the module_path Python file. The Python modules are imported using the function import_module and their contents are inspected (using inspect) to find the classes in the datamodel containing defs and with a model_to_dict method defined.

Parameters:

Name Type Description Default
module_path str

Path to the Python module file.

required

Returns:

Name Type Description
dict dict

A dictionary containing the entities in the datamodel defined in one Python module file.

Source code in bam_masterdata/metadata/entities_dict.py
def to_dict(self, module_path: str) -> dict:
    """
    Returns a dictionary containing entities read from the `module_path` Python file. The Python modules
    are imported using the function `import_module` and their contents are inspected (using `inspect`) to
    find the classes in the datamodel containing `defs` and with a `model_to_dict` method defined.

    Args:
        module_path (str): Path to the Python module file.

    Returns:
        dict: A dictionary containing the entities in the datamodel defined in one Python module file.
    """
    module = import_module(module_path=module_path)

    # initializing the dictionary with keys as the `code` of the entity and values the json dumped data
    data: dict = {}

    # Read the module source code and store line numbers
    module_source = self._read_module_source(module_path)
    class_locations = self._collect_class_locations(module_source)
    property_locations = self._collect_member_locations(
        module_source, "PropertyTypeAssignment"
    )
    vocabulary_term_locations = self._collect_member_locations(
        module_source, "VocabularyTerm"
    )

    # Process all classes in the module
    for name, obj in inspect.getmembers(module, inspect.isclass):
        if not hasattr(obj, "defs") or not callable(getattr(obj, "model_to_dict")):
            continue
        try:
            obj_data = obj().model_to_dict()
            obj_data["defs"]["row_location"] = class_locations.get(name, None)

            if "properties" in obj_data:
                self._assign_row_locations(
                    name,
                    obj_data["properties"],
                    property_locations,
                    strip_dollar=True,
                )

            elif "terms" in obj_data:
                self._assign_row_locations(
                    name, obj_data["terms"], vocabulary_term_locations
                )

            data[obj.defs.code] = obj_data
        except Exception as err:
            click.echo(f"Failed to process class {name} in {module_path}: {err}")

    return data

bam_masterdata.cli.fill_masterdata

MasterdataCodeGenerator

Class to generate Python code for the masterdata datamodel based on the entities existing in an openBIS instance.

Source code in bam_masterdata/cli/fill_masterdata.py
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
class MasterdataCodeGenerator:
    """
    Class to generate Python code for the masterdata datamodel based on the entities existing in an
    openBIS instance.
    """

    def __init__(self, url: str = "", path: str = "", **kwargs):
        start_time = time.time()
        self.row_cell_info = kwargs.get("row_cell_info", False)
        # * This part takes some time due to the loading of all entities from Openbis
        if url:
            self.generator_type = "openbis"
            self.properties = OpenbisEntities(url=url).get_property_dict()
            self.collections = OpenbisEntities(url=url).get_collection_dict()
            self.datasets = OpenbisEntities(url=url).get_dataset_dict()
            self.objects = OpenbisEntities(url=url).get_object_dict()
            self.vocabularies = OpenbisEntities(url=url).get_vocabulary_dict()
            elapsed_time = time.time() - start_time
            click.echo(
                f"Loaded OpenBIS entities in `MasterdataCodeGenerator` initialization {elapsed_time:.2f} seconds\n"
            )
        else:
            self.generator_type = "excel"
            entities_dict = MasterdataExcelExtractor(
                excel_path=path, row_cell_info=self.row_cell_info
            ).excel_to_entities()
            self.properties = entities_dict.get("property_types", {})
            self.collections = entities_dict.get("collection_types", {})
            self.datasets = entities_dict.get("dataset_types", {})
            self.objects = entities_dict.get("object_types", {})
            self.vocabularies = entities_dict.get("vocabulary_types", {})
            elapsed_time = time.time() - start_time
            click.echo(
                f"Loaded Masterdata excel entities in `MasterdataCodeGenerator` initialization {elapsed_time:.2f} seconds\n"
            )

    def determine_parent_class(
        self, code: str, class_names: dict, default: str, lines: list
    ) -> tuple:
        """
        Determine the parent class information of the entity based on its `code`. It returns
        the `parent_code` and `parent class`, as well as the `class_name` of the entity. The
        class will inherit from `parent_class`.

        If the parent class does not exist, a note is added to the `lines` list for debugging purposes.

        Args:
            code (str): The code of the entity.
            class_names (dict): A dictionary with the class names of the entities.
            default (str): The default parent class if the parent class does not exist.
            lines (list): A list of strings to be printed to the Python module.
        Returns:
            tuple: The parent code, parent class, and class name of the entity.
        """
        parent_code = ""
        if "." in code:
            parent_code = code.rsplit(".", 1)[0]
        parent_class = class_names.get(parent_code, default)

        # Format class name
        class_name = code_to_class_name(code)
        class_names[code] = class_name

        # If the parent class does not exist but the `code` shows some inheritance, we add a note for debugging
        if parent_code and parent_class == default:
            lines.append(
                f"# ! The parent class of {class_name} is not defined (missing {parent_class})"
            )

        return parent_code, parent_class, class_name

    def get_property_object_code(self, prop_data: dict) -> str:
        """
        Get the object code (or vocabulary code) used for reference for the assigned property with `prop_code`.

        Args:
            prop_data (dict): The data information for the property as obtained from openBIS.

        Returns:
            str: The object/vocabulary code used for reference for the assigned property.
        """
        if not prop_data:
            return ""

        # TODO check excel extractor to add sampleType column
        object_code = prop_data.get("sampleType", "")
        if object_code:
            return object_code

        # TODO fix this patch and avoid using generator type
        vocabulary_code = ""
        if self.generator_type == "openbis":
            vocabulary_code = prop_data.get("vocabulary", "")
        elif self.generator_type == "excel":
            vocabulary_code = prop_data.get("vocabularyType", "")
        return vocabulary_code

    def add_properties(
        self, entities: dict, parent_code: str, data: dict, lines: list
    ) -> None:
        """
        Add the properties of the entity to the `lines` list. The properties are added as
        `PropertyTypeAssignment` objects.

        Note: the assigned properties do not have the information of `code` for the entity when
        data_type is OBJECT or CONTROLLEDVOCABULARY. These are instead defined in `property_types.py`.

        Args:
            entities (dict): The dictionary of entities (objects, collections, datasets, vocabularies).
            parent_code (code): The code of the parent class.
            data (dict): The data information for the entity as obtained from openBIS.
            lines (list): A list of strings to be printed to the Python module.
        """
        parent_properties_code = (
            entities.get(parent_code, {}).get("properties", {}).keys()
        )
        for prop_code, prop_data in data.get("properties", {}).items():
            # Skip "UNKNOWN" properties
            # We check if the property is inherited from the parent class
            if prop_code == "UNKNOWN" or prop_code in parent_properties_code:
                continue

            prop_name = prop_code.lstrip("$").replace(".", "_").lower()
            lines.append(f"    {prop_name} = PropertyTypeAssignment(")
            lines.append(f'        code="{prop_code}",')
            # ! patching dataType=SAMPLE instead of OBJECT
            data_type = prop_data.get("dataType", "")
            if data_type == "SAMPLE":
                data_type = "OBJECT"
            lines.append(f'        data_type="{data_type}",')
            if data_type == "OBJECT":
                object_code = self.get_property_object_code(prop_data=prop_data)
                if object_code:
                    lines.append(f'        object_code="{object_code}",')
            elif data_type == "CONTROLLEDVOCABULARY":
                vocabulary_code = self.get_property_object_code(prop_data=prop_data)
                if vocabulary_code:
                    lines.append(f'        vocabulary_code="{vocabulary_code}",')

            property_label = (prop_data.get("label") or "").replace("\n", "\\n")
            lines.append(f'        property_label="{property_label}",')
            description = (
                (prop_data.get("description") or "")
                .replace('"', "`")
                .replace("\n", "\\n")
                .replace("'", "`")
            )
            lines.append(f'        description="""{description}""",')
            lines.append(f"        mandatory={prop_data.get('mandatory', False)},")
            lines.append(
                f"        show_in_edit_views={prop_data.get('show_in_edit_views', False)},"
            )
            section = (
                (prop_data.get("section") or "")
                .replace('"', '\\"')
                .replace("\n", "\\n")
                .replace("'", "\\'")
            )
            lines.append(f'        section="{section}",')
            lines.append("    )")
            lines.append("")

    def generate_collection_types(self) -> str:
        """
        Generate Python code for the collection types in the Openbis datamodel. The code is generated
        as a string which is then printed out to the specific Python module in `bam_masterdata/datamodel/collection_types.py`.

        Returns:
            str: Python code for the collection types.
        """
        lines = []
        class_names: dict = {}
        # from bam_masterdata.metadata.definitions import (
        #     CollectionTypeDef,
        #     PropertyTypeAssignment,
        # )
        if self.collections != {}:
            # Add imports at the top
            lines.append("from bam_masterdata.metadata.definitions import (")
            lines.append("    CollectionTypeDef,")
            lines.append("    PropertyTypeAssignment,")
            lines.append(")")
            lines.append("from bam_masterdata.metadata.entities import CollectionType")
            lines.append("")
            lines.append("")

        # Process each collection type
        for code, data in self.collections.items():
            # Skip the "UNKNOWN" object type
            if code == "UNKNOWN":
                continue

            # Determine parent class
            parent_code, parent_class, class_name = self.determine_parent_class(
                code=code,
                class_names=class_names,
                default="CollectionType",
                lines=lines,
            )

            # Add class definition
            lines.append(f"class {class_name}({parent_class}):")
            lines.append("    defs = CollectionTypeDef(")
            lines.append(f'        code="{code}",')
            description = (
                (data.get("description") or "")
                .replace('"', "`")
                .replace("\n", "\\n")
                .replace("'", "`")
            )
            lines.append(f'        description="""{description}""",')
            if data.get("validationPlugin") != "":
                lines.append(
                    f'        validation_script="{data.get("validationPlugin")}",'
                )
            lines.append("    )")
            lines.append("")

            # Add properties
            self.add_properties(self.collections, parent_code, data, lines)
            # Add newline between classes
            lines.append("")

        return "\n".join(lines)

    def generate_dataset_types(self) -> str:
        """
        Generate Python code for the dataset types in the Openbis datamodel. The code is generated
        as a string which is then printed out to the specific Python module in `bam_masterdata/datamodel/dataset_types.py`.

        Returns:
            str: Python code for the dataset types.
        """
        lines = []
        class_names: dict = {}

        if self.datasets != {}:
            # Add imports at the top
            lines.append(
                "from bam_masterdata.metadata.definitions import DatasetTypeDef, PropertyTypeAssignment"
            )
            lines.append("from bam_masterdata.metadata.entities import DatasetType")
            lines.append("")
            lines.append("")

        # Process each dataset type
        for code, data in self.datasets.items():
            # Skip the "UNKNOWN" object type
            if code == "UNKNOWN":
                continue

            # Determine parent class
            parent_code, parent_class, class_name = self.determine_parent_class(
                code=code, class_names=class_names, default="DatasetType", lines=lines
            )

            # Add class definition
            lines.append(f"class {class_name}({parent_class}):")
            lines.append("    defs = DatasetTypeDef(")
            lines.append(f'        code="{code}",')
            description = (
                (data.get("description") or "")
                .replace('"', "`")
                .replace("\n", "\\n")
                .replace("'", "`")
            )
            lines.append(f'        description="""{description}""",')
            lines.append("    )")
            lines.append("")

            # Add properties
            self.add_properties(self.datasets, parent_code, data, lines)
            # Add newline between classes
            lines.append("")

        return "\n".join(lines)

    def generate_object_types(self) -> str:
        """
        Generate Python code for the object types in the Openbis datamodel. The code is generated
        as a string which is then printed out to the specific Python module in `bam_masterdata/datamodel/object_types.py`.

        Returns:
            str: Python code for the object types.
        """
        lines = []
        class_names: dict = {}

        if self.objects != {}:
            # Add imports at the top
            lines.append(
                "from bam_masterdata.metadata.definitions import ObjectTypeDef, PropertyTypeAssignment"
            )
            lines.append("from bam_masterdata.metadata.entities import ObjectType")
            lines.append("")
            lines.append("")

        # Process each object type
        for code, data in self.objects.items():
            # Skip the "UNKNOWN" object type
            if code == "UNKNOWN":
                continue

            # Determine parent class
            parent_code, parent_class, class_name = self.determine_parent_class(
                code=code, class_names=class_names, default="ObjectType", lines=lines
            )

            # Add class definition
            lines.append(f"class {class_name}({parent_class}):")
            lines.append("    defs = ObjectTypeDef(")
            lines.append(f'        code="{code}",')
            description = (
                (data.get("description") or "")
                .replace('"', "`")
                .replace("\n", "\\n")
                .replace("'", "`")
            )
            lines.append(f'        description="""{description}""",')
            lines.append(
                f'        generated_code_prefix="{data.get("generatedCodePrefix", "")}",'
            )
            lines.append("    )")
            lines.append("")

            # Add properties
            self.add_properties(self.objects, parent_code, data, lines)
            # Add newline between classes
            lines.append("")

        return "\n".join(lines)

    def generate_vocabulary_types(self) -> str:
        """
        Generate Python code for the vocabulary types in the Openbis datamodel. The code is generated
        as a string which is then printed out to the specific Python module in `bam_masterdata/datamodel/vocabulary_types.py`.

        Returns:
            str: Python code for the vocabulary types.
        """
        lines = []
        class_names: dict = {}

        if self.vocabularies != {}:
            # Add imports at the top
            lines.append(
                "from bam_masterdata.metadata.definitions import VocabularyTerm, VocabularyTypeDef"
            )
            lines.append("from bam_masterdata.metadata.entities import VocabularyType")
            lines.append("")
            lines.append("")

        # Process each object type
        for code, data in self.vocabularies.items():
            # Skip the "UNKNOWN" object type
            if code == "UNKNOWN":
                continue

            # Determine parent class
            parent_code, parent_class, class_name = self.determine_parent_class(
                code=code,
                class_names=class_names,
                default="VocabularyType",
                lines=lines,
            )

            # Add class definition
            lines.append(f"class {class_name}({parent_class}):")
            lines.append("    defs = VocabularyTypeDef(")
            lines.append(f'        code="{code}",')
            description = (
                (data.get("description") or "")
                .replace('"', "`")
                .replace("\n", "\\n")
                .replace("'", "`")
            )
            lines.append(f'        description="""{description}""",')
            lines.append("    )")
            lines.append("")

            # Add terms
            parent_terms = self.objects.get(parent_code, {}).get("terms", {}).keys()
            for term_code, term_data in data.get("terms", {}).items():
                # Skip "UNKNOWN" properties
                if term_code == "UNKNOWN":
                    continue

                # We check if the term is inherited from the parent class
                if term_code in parent_terms:
                    continue

                term_name = (
                    term_code.lstrip("$").replace(".", "_").replace("-", "_").lower()
                )
                if term_name[0].isdigit():
                    term_name = f"_{term_name}"
                if term_name == "l":
                    term_name = "L"
                if term_name == "O":
                    term_name = "o"
                if term_name == "I":
                    term_name = "i"
                lines.append(f"    {term_name} = VocabularyTerm(")
                lines.append(f'        code="{term_code}",')
                label = (term_data.get("label") or "").replace('"', "")
                lines.append(f'        label="{label}",')
                description = (
                    (term_data.get("description") or "")
                    .replace('"', "`")
                    .replace("\n", "\\n")
                    .replace("'", "`")
                )
                lines.append(f'        description="""{description}""",')
                lines.append("    )")
                lines.append("")

            # Add newline between classes
            lines.append("")

        return "\n".join(lines)

add_properties(entities, parent_code, data, lines)

Add the properties of the entity to the lines list. The properties are added as PropertyTypeAssignment objects.

Note: the assigned properties do not have the information of code for the entity when data_type is OBJECT or CONTROLLEDVOCABULARY. These are instead defined in property_types.py.

Parameters:

Name Type Description Default
entities dict

The dictionary of entities (objects, collections, datasets, vocabularies).

required
parent_code code

The code of the parent class.

required
data dict

The data information for the entity as obtained from openBIS.

required
lines list

A list of strings to be printed to the Python module.

required
Source code in bam_masterdata/cli/fill_masterdata.py
def add_properties(
    self, entities: dict, parent_code: str, data: dict, lines: list
) -> None:
    """
    Add the properties of the entity to the `lines` list. The properties are added as
    `PropertyTypeAssignment` objects.

    Note: the assigned properties do not have the information of `code` for the entity when
    data_type is OBJECT or CONTROLLEDVOCABULARY. These are instead defined in `property_types.py`.

    Args:
        entities (dict): The dictionary of entities (objects, collections, datasets, vocabularies).
        parent_code (code): The code of the parent class.
        data (dict): The data information for the entity as obtained from openBIS.
        lines (list): A list of strings to be printed to the Python module.
    """
    parent_properties_code = (
        entities.get(parent_code, {}).get("properties", {}).keys()
    )
    for prop_code, prop_data in data.get("properties", {}).items():
        # Skip "UNKNOWN" properties
        # We check if the property is inherited from the parent class
        if prop_code == "UNKNOWN" or prop_code in parent_properties_code:
            continue

        prop_name = prop_code.lstrip("$").replace(".", "_").lower()
        lines.append(f"    {prop_name} = PropertyTypeAssignment(")
        lines.append(f'        code="{prop_code}",')
        # ! patching dataType=SAMPLE instead of OBJECT
        data_type = prop_data.get("dataType", "")
        if data_type == "SAMPLE":
            data_type = "OBJECT"
        lines.append(f'        data_type="{data_type}",')
        if data_type == "OBJECT":
            object_code = self.get_property_object_code(prop_data=prop_data)
            if object_code:
                lines.append(f'        object_code="{object_code}",')
        elif data_type == "CONTROLLEDVOCABULARY":
            vocabulary_code = self.get_property_object_code(prop_data=prop_data)
            if vocabulary_code:
                lines.append(f'        vocabulary_code="{vocabulary_code}",')

        property_label = (prop_data.get("label") or "").replace("\n", "\\n")
        lines.append(f'        property_label="{property_label}",')
        description = (
            (prop_data.get("description") or "")
            .replace('"', "`")
            .replace("\n", "\\n")
            .replace("'", "`")
        )
        lines.append(f'        description="""{description}""",')
        lines.append(f"        mandatory={prop_data.get('mandatory', False)},")
        lines.append(
            f"        show_in_edit_views={prop_data.get('show_in_edit_views', False)},"
        )
        section = (
            (prop_data.get("section") or "")
            .replace('"', '\\"')
            .replace("\n", "\\n")
            .replace("'", "\\'")
        )
        lines.append(f'        section="{section}",')
        lines.append("    )")
        lines.append("")

determine_parent_class(code, class_names, default, lines)

Determine the parent class information of the entity based on its code. It returns the parent_code and parent class, as well as the class_name of the entity. The class will inherit from parent_class.

If the parent class does not exist, a note is added to the lines list for debugging purposes.

Parameters:

Name Type Description Default
code str

The code of the entity.

required
class_names dict

A dictionary with the class names of the entities.

required
default str

The default parent class if the parent class does not exist.

required
lines list

A list of strings to be printed to the Python module.

required

Returns: tuple: The parent code, parent class, and class name of the entity.

Source code in bam_masterdata/cli/fill_masterdata.py
def determine_parent_class(
    self, code: str, class_names: dict, default: str, lines: list
) -> tuple:
    """
    Determine the parent class information of the entity based on its `code`. It returns
    the `parent_code` and `parent class`, as well as the `class_name` of the entity. The
    class will inherit from `parent_class`.

    If the parent class does not exist, a note is added to the `lines` list for debugging purposes.

    Args:
        code (str): The code of the entity.
        class_names (dict): A dictionary with the class names of the entities.
        default (str): The default parent class if the parent class does not exist.
        lines (list): A list of strings to be printed to the Python module.
    Returns:
        tuple: The parent code, parent class, and class name of the entity.
    """
    parent_code = ""
    if "." in code:
        parent_code = code.rsplit(".", 1)[0]
    parent_class = class_names.get(parent_code, default)

    # Format class name
    class_name = code_to_class_name(code)
    class_names[code] = class_name

    # If the parent class does not exist but the `code` shows some inheritance, we add a note for debugging
    if parent_code and parent_class == default:
        lines.append(
            f"# ! The parent class of {class_name} is not defined (missing {parent_class})"
        )

    return parent_code, parent_class, class_name

generate_collection_types()

Generate Python code for the collection types in the Openbis datamodel. The code is generated as a string which is then printed out to the specific Python module in bam_masterdata/datamodel/collection_types.py.

Returns:

Name Type Description
str str

Python code for the collection types.

Source code in bam_masterdata/cli/fill_masterdata.py
def generate_collection_types(self) -> str:
    """
    Generate Python code for the collection types in the Openbis datamodel. The code is generated
    as a string which is then printed out to the specific Python module in `bam_masterdata/datamodel/collection_types.py`.

    Returns:
        str: Python code for the collection types.
    """
    lines = []
    class_names: dict = {}
    # from bam_masterdata.metadata.definitions import (
    #     CollectionTypeDef,
    #     PropertyTypeAssignment,
    # )
    if self.collections != {}:
        # Add imports at the top
        lines.append("from bam_masterdata.metadata.definitions import (")
        lines.append("    CollectionTypeDef,")
        lines.append("    PropertyTypeAssignment,")
        lines.append(")")
        lines.append("from bam_masterdata.metadata.entities import CollectionType")
        lines.append("")
        lines.append("")

    # Process each collection type
    for code, data in self.collections.items():
        # Skip the "UNKNOWN" object type
        if code == "UNKNOWN":
            continue

        # Determine parent class
        parent_code, parent_class, class_name = self.determine_parent_class(
            code=code,
            class_names=class_names,
            default="CollectionType",
            lines=lines,
        )

        # Add class definition
        lines.append(f"class {class_name}({parent_class}):")
        lines.append("    defs = CollectionTypeDef(")
        lines.append(f'        code="{code}",')
        description = (
            (data.get("description") or "")
            .replace('"', "`")
            .replace("\n", "\\n")
            .replace("'", "`")
        )
        lines.append(f'        description="""{description}""",')
        if data.get("validationPlugin") != "":
            lines.append(
                f'        validation_script="{data.get("validationPlugin")}",'
            )
        lines.append("    )")
        lines.append("")

        # Add properties
        self.add_properties(self.collections, parent_code, data, lines)
        # Add newline between classes
        lines.append("")

    return "\n".join(lines)

generate_dataset_types()

Generate Python code for the dataset types in the Openbis datamodel. The code is generated as a string which is then printed out to the specific Python module in bam_masterdata/datamodel/dataset_types.py.

Returns:

Name Type Description
str str

Python code for the dataset types.

Source code in bam_masterdata/cli/fill_masterdata.py
def generate_dataset_types(self) -> str:
    """
    Generate Python code for the dataset types in the Openbis datamodel. The code is generated
    as a string which is then printed out to the specific Python module in `bam_masterdata/datamodel/dataset_types.py`.

    Returns:
        str: Python code for the dataset types.
    """
    lines = []
    class_names: dict = {}

    if self.datasets != {}:
        # Add imports at the top
        lines.append(
            "from bam_masterdata.metadata.definitions import DatasetTypeDef, PropertyTypeAssignment"
        )
        lines.append("from bam_masterdata.metadata.entities import DatasetType")
        lines.append("")
        lines.append("")

    # Process each dataset type
    for code, data in self.datasets.items():
        # Skip the "UNKNOWN" object type
        if code == "UNKNOWN":
            continue

        # Determine parent class
        parent_code, parent_class, class_name = self.determine_parent_class(
            code=code, class_names=class_names, default="DatasetType", lines=lines
        )

        # Add class definition
        lines.append(f"class {class_name}({parent_class}):")
        lines.append("    defs = DatasetTypeDef(")
        lines.append(f'        code="{code}",')
        description = (
            (data.get("description") or "")
            .replace('"', "`")
            .replace("\n", "\\n")
            .replace("'", "`")
        )
        lines.append(f'        description="""{description}""",')
        lines.append("    )")
        lines.append("")

        # Add properties
        self.add_properties(self.datasets, parent_code, data, lines)
        # Add newline between classes
        lines.append("")

    return "\n".join(lines)

generate_object_types()

Generate Python code for the object types in the Openbis datamodel. The code is generated as a string which is then printed out to the specific Python module in bam_masterdata/datamodel/object_types.py.

Returns:

Name Type Description
str str

Python code for the object types.

Source code in bam_masterdata/cli/fill_masterdata.py
def generate_object_types(self) -> str:
    """
    Generate Python code for the object types in the Openbis datamodel. The code is generated
    as a string which is then printed out to the specific Python module in `bam_masterdata/datamodel/object_types.py`.

    Returns:
        str: Python code for the object types.
    """
    lines = []
    class_names: dict = {}

    if self.objects != {}:
        # Add imports at the top
        lines.append(
            "from bam_masterdata.metadata.definitions import ObjectTypeDef, PropertyTypeAssignment"
        )
        lines.append("from bam_masterdata.metadata.entities import ObjectType")
        lines.append("")
        lines.append("")

    # Process each object type
    for code, data in self.objects.items():
        # Skip the "UNKNOWN" object type
        if code == "UNKNOWN":
            continue

        # Determine parent class
        parent_code, parent_class, class_name = self.determine_parent_class(
            code=code, class_names=class_names, default="ObjectType", lines=lines
        )

        # Add class definition
        lines.append(f"class {class_name}({parent_class}):")
        lines.append("    defs = ObjectTypeDef(")
        lines.append(f'        code="{code}",')
        description = (
            (data.get("description") or "")
            .replace('"', "`")
            .replace("\n", "\\n")
            .replace("'", "`")
        )
        lines.append(f'        description="""{description}""",')
        lines.append(
            f'        generated_code_prefix="{data.get("generatedCodePrefix", "")}",'
        )
        lines.append("    )")
        lines.append("")

        # Add properties
        self.add_properties(self.objects, parent_code, data, lines)
        # Add newline between classes
        lines.append("")

    return "\n".join(lines)

generate_vocabulary_types()

Generate Python code for the vocabulary types in the Openbis datamodel. The code is generated as a string which is then printed out to the specific Python module in bam_masterdata/datamodel/vocabulary_types.py.

Returns:

Name Type Description
str str

Python code for the vocabulary types.

Source code in bam_masterdata/cli/fill_masterdata.py
def generate_vocabulary_types(self) -> str:
    """
    Generate Python code for the vocabulary types in the Openbis datamodel. The code is generated
    as a string which is then printed out to the specific Python module in `bam_masterdata/datamodel/vocabulary_types.py`.

    Returns:
        str: Python code for the vocabulary types.
    """
    lines = []
    class_names: dict = {}

    if self.vocabularies != {}:
        # Add imports at the top
        lines.append(
            "from bam_masterdata.metadata.definitions import VocabularyTerm, VocabularyTypeDef"
        )
        lines.append("from bam_masterdata.metadata.entities import VocabularyType")
        lines.append("")
        lines.append("")

    # Process each object type
    for code, data in self.vocabularies.items():
        # Skip the "UNKNOWN" object type
        if code == "UNKNOWN":
            continue

        # Determine parent class
        parent_code, parent_class, class_name = self.determine_parent_class(
            code=code,
            class_names=class_names,
            default="VocabularyType",
            lines=lines,
        )

        # Add class definition
        lines.append(f"class {class_name}({parent_class}):")
        lines.append("    defs = VocabularyTypeDef(")
        lines.append(f'        code="{code}",')
        description = (
            (data.get("description") or "")
            .replace('"', "`")
            .replace("\n", "\\n")
            .replace("'", "`")
        )
        lines.append(f'        description="""{description}""",')
        lines.append("    )")
        lines.append("")

        # Add terms
        parent_terms = self.objects.get(parent_code, {}).get("terms", {}).keys()
        for term_code, term_data in data.get("terms", {}).items():
            # Skip "UNKNOWN" properties
            if term_code == "UNKNOWN":
                continue

            # We check if the term is inherited from the parent class
            if term_code in parent_terms:
                continue

            term_name = (
                term_code.lstrip("$").replace(".", "_").replace("-", "_").lower()
            )
            if term_name[0].isdigit():
                term_name = f"_{term_name}"
            if term_name == "l":
                term_name = "L"
            if term_name == "O":
                term_name = "o"
            if term_name == "I":
                term_name = "i"
            lines.append(f"    {term_name} = VocabularyTerm(")
            lines.append(f'        code="{term_code}",')
            label = (term_data.get("label") or "").replace('"', "")
            lines.append(f'        label="{label}",')
            description = (
                (term_data.get("description") or "")
                .replace('"', "`")
                .replace("\n", "\\n")
                .replace("'", "`")
            )
            lines.append(f'        description="""{description}""",')
            lines.append("    )")
            lines.append("")

        # Add newline between classes
        lines.append("")

    return "\n".join(lines)

get_property_object_code(prop_data)

Get the object code (or vocabulary code) used for reference for the assigned property with prop_code.

Parameters:

Name Type Description Default
prop_data dict

The data information for the property as obtained from openBIS.

required

Returns:

Name Type Description
str str

The object/vocabulary code used for reference for the assigned property.

Source code in bam_masterdata/cli/fill_masterdata.py
def get_property_object_code(self, prop_data: dict) -> str:
    """
    Get the object code (or vocabulary code) used for reference for the assigned property with `prop_code`.

    Args:
        prop_data (dict): The data information for the property as obtained from openBIS.

    Returns:
        str: The object/vocabulary code used for reference for the assigned property.
    """
    if not prop_data:
        return ""

    # TODO check excel extractor to add sampleType column
    object_code = prop_data.get("sampleType", "")
    if object_code:
        return object_code

    # TODO fix this patch and avoid using generator type
    vocabulary_code = ""
    if self.generator_type == "openbis":
        vocabulary_code = prop_data.get("vocabulary", "")
    elif self.generator_type == "excel":
        vocabulary_code = prop_data.get("vocabularyType", "")
    return vocabulary_code

bam_masterdata.excel.excel_to_entities

MasterdataExcelExtractor

Source code in bam_masterdata/excel/excel_to_entities.py
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
class MasterdataExcelExtractor:
    # TODO move these validation rules to a separate json
    VALIDATION_RULES: dict[str, dict[str, dict[str, Any]]] = {}

    def __init__(self, excel_path: str, **kwargs):
        """Initialize the MasterdataExtractor."""
        self.excel_path = excel_path
        self.row_cell_info = kwargs.get("row_cell_info", False)
        self.workbook = openpyxl.load_workbook(excel_path)
        self.logger = kwargs.get("logger", logger)

        # Load validation rules at initialization
        if not MasterdataExcelExtractor.VALIDATION_RULES:
            self.VALIDATION_RULES = load_validation_rules(
                self.logger,
                os.path.join(VALIDATION_RULES_DIR, "excel_validation_rules.json"),
            )

    def index_to_excel_column(self, index: int) -> str:
        """
        Converts a 1-based index to an Excel column name.

        Args:
            index: The 1-based index to convert.

        Returns:
            The corresponding Excel column name.
        """
        if not index >= 1:
            raise ValueError("Index must be a positive integer starting from 1.")

        column = ""
        while index > 0:
            index, remainder = divmod(index - 1, 26)
            column = chr(65 + remainder) + column
        return column

    def get_last_non_empty_row(
        self, sheet: "Worksheet", start_index: int
    ) -> int | None:
        """
        Finds the last non-empty row before encountering a completely empty row.

        Args:
            sheet: The worksheet object.
            start_index: The row number to start checking from (1-based index).

        Returns:
            The row number of the last non-empty row before an empty row is encountered,
            or None if no non-empty rows are found starting from the given index.
        """
        if start_index < 1 or start_index > sheet.max_row:
            raise ValueError(
                f"Invalid start index: {start_index}. It must be between 1 and {sheet.max_row}."
            )

        last_non_empty_row = None
        for row in range(start_index, sheet.max_row + 1):
            if all(
                sheet.cell(row=row, column=col).value in (None, "")
                for col in range(1, sheet.max_column + 1)
            ):
                return last_non_empty_row  # Return the last non-empty row before the current empty row

            last_non_empty_row = row  # Update the last non-empty row

        return last_non_empty_row  # If no empty row is encountered, return the last non-empty row

    def str_to_bool(
        self,
        value: str | bool | None,
        term: str,
        coordinate: str,
        sheet_title: str,
    ) -> bool:
        """
        Converts a string to a boolean value.

        Args:
            value: The string to convert.

        Returns:
            The boolean value.
        """
        # No `value` provided
        if not value:
            return False

        val = str(value).strip().lower()
        if val not in ["true", "false"]:
            self.logger.error(
                f"Invalid {term.lower()} value found in the {term} column at position {coordinate} in {sheet_title}. Accepted values: TRUE or FALSE.",
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
        return val == "true"

    def get_and_check_property(
        self,
        value: str | bool | None,
        term: str,
        coordinate: str,
        sheet_title: str,
        is_description: bool = False,
        is_code: bool = False,
        is_data: bool = False,
        is_url: bool = False,
    ) -> str:
        """
        Gets a property and checks its format.

        Args:
            value: The string to convert.

        Returns:
            The property.
        """

        # No `value` provided
        if not value:
            return ""

        val = str(value)
        error_message = f"Invalid {term.lower()} value found in the {term} column at position {coordinate} in {sheet_title}."
        if is_description:
            if not re.match(r".*//.*", val):
                self.logger.error(
                    error_message
                    + "Description should follow the schema: English Description + '//' + German Description. ",
                    term=term,
                    cell_value=val,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
        elif is_code:
            if not re.match(r"^\$?[A-Z0-9_.]+$", val):
                self.logger.error(
                    error_message,
                    term=term,
                    cell_value=val,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
        elif is_data:
            # Normalize data type to uppercase and allow dynamic SAMPLE/OBJECT codes
            val_upper = val.upper()
            allowed_types = [dt.value for dt in DataType]
            is_valid_standard = val_upper in allowed_types
            is_valid_dynamic = False

            if not is_valid_standard and (
                val_upper.startswith("SAMPLE:") or val_upper.startswith("OBJECT:")
            ):
                parts = val_upper.split(":", 1)
                if len(parts) == 2 and parts[1].strip():
                    is_valid_dynamic = True

            if not is_valid_standard and not is_valid_dynamic:
                self.logger.error(
                    error_message
                    + f"The Data Type should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                    term=term,
                    cell_value=val_upper,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
            val = val_upper
        elif is_url:
            if not re.match(
                r"https?://(?:www\.)?[a-zA-Z0-9-._~:/?#@!$&'()*+,;=%]+", val
            ):
                self.logger.error(
                    error_message,
                    term=term,
                    cell_value=val,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
        else:
            if not re.match(r".*", val):
                self.logger.error(
                    error_message,
                    term=term,
                    cell_value=val,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
        return val

    # Helper function to process each term
    def process_term(
        self, term: str, cell_value: Any, coordinate: str, sheet_title: str
    ) -> Any:
        """
        Processes a term by converting it to a boolean if necessary or checking its validity.

        Args:
            term: The term being processed.
            cell_value: The value of the cell.
            coordinate: The coordinate of the cell in the sheet.
            sheet_title: The title of the sheet.

        Returns:
            The processed value, either as a boolean or the original value after validation.
        """
        # Check if the term is a boolean type
        if term in ("Mandatory", "Show in edit views"):
            return self.str_to_bool(
                value=cell_value,
                term=term,
                coordinate=coordinate,
                sheet_title=sheet_title,
            )
        # Check and validate the property
        return self.get_and_check_property(
            value=cell_value,
            term=term,
            coordinate=coordinate,
            sheet_title=sheet_title,
            is_code=(term in ["Code", "Vocabulary code"]),
            is_data=(term == "Data type"),
        )

    def extract_value(
        self,
        sheet: "Worksheet",
        row: int,
        column: int,
        validation_pattern: str = None,
        is_description: bool = False,
        is_data: bool = False,
        is_url: bool = False,
    ) -> str:
        """
        Extracts and validates a value from a specified cell in the Excel sheet.

        Args:
            sheet: The worksheet object.
            row: The row number of the cell (1-based index).
            column: The column number of the cell (1-based index).
            validation_pattern: Optional regex pattern to validate the cell value.
            is_description: Flag indicating if the value is a description.
            is_data: Flag indicating if the value is a data type.
            is_url: Flag indicating if the value is a URL.

        Returns:
            The extracted and validated cell value as a string. Returns an empty string if the value is invalid or not provided.
        """
        value = sheet.cell(row=row, column=column).value

        # No `value` provided
        if not value:
            return ""

        validated = (
            bool(re.match(validation_pattern, str(value)))
            if validation_pattern
            else True
        )
        error_message = f"Invalid value '{value}' at row {row}, column {column} in sheet {sheet.title}"

        if is_description:
            error_message += " Description should follow the schema: English Description + '//' + German Description."
        elif is_data:
            val_upper = str(value).upper()
            allowed_types = [dt.value for dt in DataType]
            is_valid_standard = val_upper in allowed_types
            is_valid_dynamic = False
            if not is_valid_standard and (
                val_upper.startswith("SAMPLE:") or val_upper.startswith("OBJECT:")
            ):
                parts = val_upper.split(":", 1)
                if len(parts) == 2 and parts[1].strip():
                    is_valid_dynamic = True
            validated = is_valid_standard or is_valid_dynamic
            error_message += f" The Data Type should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'"
        elif is_url:
            error_message += " It should be an URL or empty"

        if not validated:
            self.logger.error(
                error_message,
                cell_value=value,
                sheet_title=sheet.title,
                row=row,
                column=column,
            )

        return value or ""

    def process_entity(
        self,
        sheet: "Worksheet",
        start_index_row: int,
        header_terms: list[str],
        expected_terms: list[str],
        entity_type: str,
    ) -> dict[str, Any]:
        """
        Process an entity type block in the Excel sheet and return its attributes as a dictionary.

        Args:
            sheet: The worksheet object.
            start_index_row: The row where the current entity type begins (1-based index).
            header_terms: List of header terms in the entity block.
            expected_terms: List of expected terms to extract from the entity block.
            entity_type: The type of the entity (e.g., SAMPLE_TYPE, OBJECT_TYPE).

        Returns:
            A dictionary containing the attributes of the entity.
        """
        attributes: dict = {}
        cell_value: Any = ""

        for term in expected_terms:
            if term not in header_terms:
                self.logger.error(f"{term} not found in the headers.", term=term)
            else:
                term_index = header_terms.index(term)
                cell = sheet.cell(row=start_index_row + 2, column=term_index + 1)
                cell_value = self.extract_value(
                    sheet,
                    start_index_row + 2,
                    term_index + 1,
                    self.VALIDATION_RULES[entity_type][term].get("pattern"),
                )

                # Handle boolean conversion
                if self.VALIDATION_RULES[entity_type][term].get("is_bool"):
                    cell_value = self.str_to_bool(
                        value=cell_value,
                        term=term,
                        coordinate=cell.coordinate,
                        sheet_title=sheet.title,
                    )

                # Handle data type validation
                elif self.VALIDATION_RULES[entity_type][term].get("is_data"):
                    allowed_types = [dt.value for dt in DataType]
                    cell_value_upper = str(cell_value).upper()
                    is_valid_standard = cell_value_upper in allowed_types
                    is_valid_dynamic = False
                    if not is_valid_standard and (
                        cell_value_upper.startswith("SAMPLE:")
                        or cell_value_upper.startswith("OBJECT:")
                    ):
                        parts = cell_value_upper.split(":", 1)
                        if len(parts) == 2 and parts[1].strip():
                            is_valid_dynamic = True
                    if not is_valid_standard and not is_valid_dynamic:
                        self.logger.error(
                            f"Invalid Data Type: {cell_value} in {cell.coordinate} (Sheet: {sheet.title}). Should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                            term=term,
                            cell_value=cell_value,
                            cell_coordinate=cell.coordinate,
                            sheet_title=sheet.title,
                        )
                    else:
                        cell_value = (
                            cell_value_upper
                            if isinstance(cell_value, str)
                            else cell_value
                        )

                # Handle additional validation for "Generated code prefix"
                elif (
                    self.VALIDATION_RULES[entity_type][term].get("extra_validation")
                    == "is_reduced_version"
                ):
                    if not is_reduced_version(cell_value, attributes.get("code", "")):
                        self.logger.warning(
                            f"Invalid {term} value '{cell_value}' in {cell.coordinate} (Sheet: {sheet.title}). "
                            f"Generated code prefix should be part of the 'Code' {attributes.get('code', '')}.",
                            term=term,
                            cell_value=cell_value,
                            cell_coordinate=cell.coordinate,
                            sheet_title=sheet.title,
                        )

                # Handle validation script (allows empty but must match pattern if provided)
                elif (
                    self.VALIDATION_RULES[entity_type][term].get("allow_empty")
                    and not cell_value
                ):
                    cell_value = None

                # Handle URL template validation (allows empty but must be a valid URL)
                elif (
                    self.VALIDATION_RULES[entity_type][term].get("is_url")
                    and cell_value
                ):
                    url_pattern = self.VALIDATION_RULES[entity_type][term].get(
                        "pattern"
                    )
                    if not re.match(url_pattern, str(cell_value)):
                        self.logger.error(
                            f"Invalid URL format: {cell_value} in {cell.coordinate} (Sheet: {sheet.title})",
                            cell_value=cell_value,
                            cell_coordinate=cell.coordinate,
                            sheet_title=sheet.title,
                        )

                # Add the extracted value to the attributes dictionary
                attributes[self.VALIDATION_RULES[entity_type][term].get("key")] = (
                    cell_value
                )

        if self.row_cell_info:
            attributes["row_location"] = f"A{start_index_row}"
        return attributes

    def properties_to_dict(
        self, sheet: "Worksheet", start_index_row: int, last_non_empty_row: int
    ) -> dict[str, dict[str, Any]]:
        """
        Extracts properties from an Entity type block in the Excel sheet and returns them as a dictionary.

        Args:
            sheet: The worksheet object.
            start_index_row: Row where the current entity type begins (1-based index).
            last_non_empty_row: Row where the current entity type finish (1-based index).

        Returns:
            A dictionary where each key is a property code and the value is a dictionary
            containing the attributes of the property.
        """
        property_dict: dict = {}
        expected_terms = [
            "Code",
            "Description",
            "Mandatory",
            "Show in edit views",
            "Section",
            "Property label",
            "Data type",
            "Vocabulary code",
            "Metadata",
            "Dynamic script",
            # ! these are not used
            # "Unique",
            # "Internal assignment",
        ]

        # Determine the header row index
        header_index = start_index_row + 3
        row_headers = [(cell.value, cell.coordinate) for cell in sheet[header_index]]
        # And store how many properties are for the entity
        n_properties = last_non_empty_row - header_index
        if n_properties < 0:
            self.logger.error(
                f"No properties found for the entity in sheet {sheet.title} starting at row {start_index_row}."
            )
            return property_dict

        # Initialize a dictionary to store extracted columns
        extracted_columns: dict[str, list] = {term: [] for term in expected_terms}
        if self.row_cell_info:
            extracted_columns["row_location"] = []

        # Extract columns for each expected term
        for i, (term, coordinate) in enumerate(row_headers):
            if term not in expected_terms:
                log_func = (
                    self.logger.warning
                    if term
                    in (
                        "Mandatory",
                        "Show in edit views",
                        "Section",
                        "Metadata",
                        "Dynamic script",
                        "Vocabulary code",
                        # ! these are not used
                        # "Unique",
                        # "Internal assignment",
                    )
                    else self.logger.error
                )
                log_func(f"'{term}' not found in the properties headers.", term=term)
                continue

            # Excel column letter from the coordinate
            term_letter = coordinate[0]

            # Extract values from the column
            for cell_property in sheet[term_letter][header_index:last_non_empty_row]:
                extracted_columns[term].append(
                    self.process_term(
                        term, cell_property.value, cell_property.coordinate, sheet.title
                    )
                )
                if self.row_cell_info:
                    extracted_columns["row_location"].append(cell_property.coordinate)

        # Combine extracted values into a dictionary
        for i in range(n_properties):
            code = extracted_columns.get("Code", [])
            if not code:
                self.logger.error(
                    f"'Code' not found in the properties headers for sheet {sheet.title}."
                )
                return property_dict
            code = code[i]
            property_dict[code] = {"permId": code, "code": code}
            for key, pybis_val in {
                "Description": "description",
                "Section": "section",
                "Mandatory": "mandatory",
                "Show in edit views": "show_in_edit_views",
                "Property label": "label",
                "Data type": "dataType",
                "Vocabulary code": "vocabularyCode",
            }.items():
                data_column = extracted_columns.get(key, [])
                if not data_column:
                    continue
                cell_value = data_column[i]
                if key == "Data type":
                    object_code = None
                    normalized_value = (
                        str(cell_value).upper()
                        if isinstance(cell_value, str)
                        else cell_value
                    )
                    if isinstance(normalized_value, str) and ":" in normalized_value:
                        prefix, dynamic_code = normalized_value.split(":", 1)
                        if prefix in ("SAMPLE", "OBJECT") and dynamic_code.strip():
                            object_code = dynamic_code.strip()
                            normalized_value = DataType.OBJECT.value
                    property_dict[code][pybis_val] = normalized_value
                    if object_code:
                        property_dict[code]["objectCode"] = object_code
                else:
                    property_dict[code][pybis_val] = cell_value
            if self.row_cell_info:
                property_dict[code]["row_location"] = (
                    extracted_columns.get("row_location")[i],
                )
            # Only add optional fields if they exist in extracted_columns
            optional_fields = [
                "Metadata",
                "Dynamic script",
                "Unique",
                "Internal assignment",
            ]
            for field in optional_fields:
                if (
                    field in extracted_columns
                ):  # Check if the field exists in the extracted columns
                    if extracted_columns[field][i] == "":
                        extracted_columns[field][i] = None
                    property_dict[extracted_columns["Code"][i]][
                        field.lower().replace(" ", "_")
                    ] = extracted_columns[field][i]

        return property_dict

    def terms_to_dict(
        self, sheet: "Worksheet", start_index_row: int, last_non_empty_row: int
    ) -> dict[str, dict[str, Any]]:
        """
        Extracts terms from a Vocabulary block in the Excel sheet and returns them as a dictionary.

        Args:
            sheet: The worksheet object.
            start_index_row: Row where the current entity type begins (1-based index).
            last_non_empty_row: Row where the current entity type finish (1-based index).

        Returns:
            A dictionary where each key is a vocabulary term code and the value is a dictionary
            containing the attributes of the vocabulary term.
        """
        terms_dict = {}
        expected_terms = ["Code", "Description", "Url template", "Label", "Official"]

        header_index = start_index_row + 3
        row_headers = [cell.value for cell in sheet[header_index]]

        # Initialize a dictionary to store extracted columns
        extracted_columns: dict[str, list] = {term: [] for term in expected_terms}

        # Helper function to process each term
        def process_term_cell(term, cell_value, coordinate, sheet_title):
            if term == "Official":
                return self.str_to_bool(
                    value=cell_value,
                    term=term,
                    coordinate=coordinate,
                    sheet_title=sheet_title,
                )
            return self.get_and_check_property(
                value=cell_value,
                term=term,
                coordinate=coordinate,
                sheet_title=sheet_title,
                is_code=(term == "Code"),
                is_url=(term == "Url template"),
            )

        # Extract columns for each expected term
        for term in expected_terms:
            if term not in row_headers:
                self.logger.warning(
                    f"{term} not found in the properties headers.", term=term
                )
                continue

            # Get column index and Excel letter
            term_index = row_headers.index(term) + 1
            term_letter = self.index_to_excel_column(term_index)

            # Extract values from the column
            for cell in sheet[term_letter][header_index:last_non_empty_row]:
                extracted_columns[term].append(
                    process_term_cell(term, cell.value, cell.coordinate, sheet.title)
                )

        if not extracted_columns.get("Code"):
            self.logger.error(
                f"The required 'Code' column for terms was not found in sheet {sheet.title}."
            )
            return {}

        # Combine extracted values into a dictionary safely
        for i in range(len(extracted_columns["Code"])):
            code = extracted_columns["Code"][i]
            terms_dict[code] = {
                "permId": code,
                "code": code,
            }
            for key, pybis_val in {
                "Description": "descriptions",
                "Url template": "url_template",
                "Label": "label",
                "Official": "official",
            }.items():
                values = extracted_columns.get(key, [])
                if len(values) > i:
                    terms_dict[code][pybis_val] = values[i]

        return terms_dict

    def block_to_entity_dict(
        self,
        sheet: "Worksheet",
        start_index_row: int,
        last_non_empty_row: int,
        complete_dict: dict[str, Any],
    ) -> dict[str, Any]:
        """
        Extracts entity attributes from an Excel sheet block and returns them as a dictionary.
        """
        attributes_dict: dict = {}

        # Get the entity type
        entity_type = sheet[f"A{start_index_row}"].value
        if entity_type not in self.VALIDATION_RULES:
            raise ValueError(f"Invalid entity type: {entity_type}")

        # Get the header terms
        header_terms = [cell.value for cell in sheet[start_index_row + 1]]

        # Process entity data using the helper function
        attributes_dict = self.process_entity(
            sheet,
            start_index_row,
            header_terms,
            list(self.VALIDATION_RULES[entity_type].keys()),
            entity_type,
        )

        # Extract additional attributes if necessary
        if entity_type in {
            "SAMPLE_TYPE",
            "OBJECT_TYPE",
            "EXPERIMENT_TYPE",
            "DATASET_TYPE",
        }:
            attributes_dict["properties"] = (
                self.properties_to_dict(sheet, start_index_row, last_non_empty_row)
                or {}
            )

        elif entity_type == "VOCABULARY_TYPE":
            attributes_dict["terms"] = (
                self.terms_to_dict(sheet, start_index_row, last_non_empty_row) or {}
            )

        # Add the entity to the complete dictionary
        complete_dict[attributes_dict["code"]] = attributes_dict

        # Return sorted dictionary
        return dict(sorted(complete_dict.items(), key=lambda item: item[0].count(".")))

    def excel_to_entities(self) -> dict[str, dict[str, Any]]:
        """
        Extracts entities from an Excel file and returns them as a dictionary.

        Returns:
            dict[str, dict[str, Any]]: A dictionary where each key is a normalized sheet name and the value is a dictionary
            containing the extracted entities. Returns an empty dictionary if all sheets are empty.
        """
        sheets_dict: dict[str, dict[str, Any]] = {}
        sheet_names = self.workbook.sheetnames
        has_content = False  # Track if any sheet has valid content

        for i, sheet_name in enumerate(sheet_names):
            normalized_sheet_name = sheet_name.lower().replace(" ", "_")
            sheet = self.workbook[sheet_name]
            start_row = 1

            # **Check if the sheet is empty**
            if all(
                sheet.cell(row=row, column=col).value in (None, "")
                for row in range(1, sheet.max_row + 1)
                for col in range(1, sheet.max_column + 1)
            ):
                self.logger.info(f"Skipping empty sheet: {sheet_name}")
                continue  # Move to the next sheet

            sheets_dict[normalized_sheet_name] = {}

            consecutive_empty_rows = 0  # Track consecutive empty rows
            while start_row <= sheet.max_row:
                # **Check for two consecutive empty rows**
                is_row_empty = all(
                    sheet.cell(row=start_row, column=col).value in (None, "")
                    for col in range(1, sheet.max_column + 1)
                )

                if is_row_empty:
                    consecutive_empty_rows += 1
                    if consecutive_empty_rows >= 2:
                        # **Reached the end of the sheet, move to the next**
                        if i == len(sheet_names) - 1:
                            self.logger.info(
                                f"Last sheet {sheet_name} processed. End of the file reached."
                            )
                        else:
                            self.logger.info(
                                f"End of the current sheet {sheet_name} reached. Switching to next sheet..."
                            )
                        break  # Stop processing this sheet
                else:
                    consecutive_empty_rows = 0  # Reset if we find a non-empty row

                    # **Process the entity block**
                    last_non_empty_row = self.get_last_non_empty_row(sheet, start_row)
                    if last_non_empty_row is None:
                        break  # No more valid blocks

                    sheets_dict[normalized_sheet_name] = self.block_to_entity_dict(
                        sheet,
                        start_row,
                        last_non_empty_row,
                        sheets_dict[normalized_sheet_name],
                    )
                    has_content = True  # Found valid content

                    # Move to the next entity block
                    start_row = last_non_empty_row + 1
                    continue  # Continue loop without increasing consecutive_empty_rows

                start_row += 1  # Move to the next row

        # **If no sheets had content, return an empty dictionary**
        if not has_content:
            self.logger.warning(
                "No valid data found in any sheets. Returning empty dictionary."
            )
            return {}

        return sheets_dict

__init__(excel_path, **kwargs)

Initialize the MasterdataExtractor.

Source code in bam_masterdata/excel/excel_to_entities.py
def __init__(self, excel_path: str, **kwargs):
    """Initialize the MasterdataExtractor."""
    self.excel_path = excel_path
    self.row_cell_info = kwargs.get("row_cell_info", False)
    self.workbook = openpyxl.load_workbook(excel_path)
    self.logger = kwargs.get("logger", logger)

    # Load validation rules at initialization
    if not MasterdataExcelExtractor.VALIDATION_RULES:
        self.VALIDATION_RULES = load_validation_rules(
            self.logger,
            os.path.join(VALIDATION_RULES_DIR, "excel_validation_rules.json"),
        )

block_to_entity_dict(sheet, start_index_row, last_non_empty_row, complete_dict)

Extracts entity attributes from an Excel sheet block and returns them as a dictionary.

Source code in bam_masterdata/excel/excel_to_entities.py
def block_to_entity_dict(
    self,
    sheet: "Worksheet",
    start_index_row: int,
    last_non_empty_row: int,
    complete_dict: dict[str, Any],
) -> dict[str, Any]:
    """
    Extracts entity attributes from an Excel sheet block and returns them as a dictionary.
    """
    attributes_dict: dict = {}

    # Get the entity type
    entity_type = sheet[f"A{start_index_row}"].value
    if entity_type not in self.VALIDATION_RULES:
        raise ValueError(f"Invalid entity type: {entity_type}")

    # Get the header terms
    header_terms = [cell.value for cell in sheet[start_index_row + 1]]

    # Process entity data using the helper function
    attributes_dict = self.process_entity(
        sheet,
        start_index_row,
        header_terms,
        list(self.VALIDATION_RULES[entity_type].keys()),
        entity_type,
    )

    # Extract additional attributes if necessary
    if entity_type in {
        "SAMPLE_TYPE",
        "OBJECT_TYPE",
        "EXPERIMENT_TYPE",
        "DATASET_TYPE",
    }:
        attributes_dict["properties"] = (
            self.properties_to_dict(sheet, start_index_row, last_non_empty_row)
            or {}
        )

    elif entity_type == "VOCABULARY_TYPE":
        attributes_dict["terms"] = (
            self.terms_to_dict(sheet, start_index_row, last_non_empty_row) or {}
        )

    # Add the entity to the complete dictionary
    complete_dict[attributes_dict["code"]] = attributes_dict

    # Return sorted dictionary
    return dict(sorted(complete_dict.items(), key=lambda item: item[0].count(".")))

excel_to_entities()

Extracts entities from an Excel file and returns them as a dictionary.

Returns:

Type Description
dict[str, dict[str, Any]]

dict[str, dict[str, Any]]: A dictionary where each key is a normalized sheet name and the value is a dictionary

dict[str, dict[str, Any]]

containing the extracted entities. Returns an empty dictionary if all sheets are empty.

Source code in bam_masterdata/excel/excel_to_entities.py
def excel_to_entities(self) -> dict[str, dict[str, Any]]:
    """
    Extracts entities from an Excel file and returns them as a dictionary.

    Returns:
        dict[str, dict[str, Any]]: A dictionary where each key is a normalized sheet name and the value is a dictionary
        containing the extracted entities. Returns an empty dictionary if all sheets are empty.
    """
    sheets_dict: dict[str, dict[str, Any]] = {}
    sheet_names = self.workbook.sheetnames
    has_content = False  # Track if any sheet has valid content

    for i, sheet_name in enumerate(sheet_names):
        normalized_sheet_name = sheet_name.lower().replace(" ", "_")
        sheet = self.workbook[sheet_name]
        start_row = 1

        # **Check if the sheet is empty**
        if all(
            sheet.cell(row=row, column=col).value in (None, "")
            for row in range(1, sheet.max_row + 1)
            for col in range(1, sheet.max_column + 1)
        ):
            self.logger.info(f"Skipping empty sheet: {sheet_name}")
            continue  # Move to the next sheet

        sheets_dict[normalized_sheet_name] = {}

        consecutive_empty_rows = 0  # Track consecutive empty rows
        while start_row <= sheet.max_row:
            # **Check for two consecutive empty rows**
            is_row_empty = all(
                sheet.cell(row=start_row, column=col).value in (None, "")
                for col in range(1, sheet.max_column + 1)
            )

            if is_row_empty:
                consecutive_empty_rows += 1
                if consecutive_empty_rows >= 2:
                    # **Reached the end of the sheet, move to the next**
                    if i == len(sheet_names) - 1:
                        self.logger.info(
                            f"Last sheet {sheet_name} processed. End of the file reached."
                        )
                    else:
                        self.logger.info(
                            f"End of the current sheet {sheet_name} reached. Switching to next sheet..."
                        )
                    break  # Stop processing this sheet
            else:
                consecutive_empty_rows = 0  # Reset if we find a non-empty row

                # **Process the entity block**
                last_non_empty_row = self.get_last_non_empty_row(sheet, start_row)
                if last_non_empty_row is None:
                    break  # No more valid blocks

                sheets_dict[normalized_sheet_name] = self.block_to_entity_dict(
                    sheet,
                    start_row,
                    last_non_empty_row,
                    sheets_dict[normalized_sheet_name],
                )
                has_content = True  # Found valid content

                # Move to the next entity block
                start_row = last_non_empty_row + 1
                continue  # Continue loop without increasing consecutive_empty_rows

            start_row += 1  # Move to the next row

    # **If no sheets had content, return an empty dictionary**
    if not has_content:
        self.logger.warning(
            "No valid data found in any sheets. Returning empty dictionary."
        )
        return {}

    return sheets_dict

extract_value(sheet, row, column, validation_pattern=None, is_description=False, is_data=False, is_url=False)

Extracts and validates a value from a specified cell in the Excel sheet.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
row int

The row number of the cell (1-based index).

required
column int

The column number of the cell (1-based index).

required
validation_pattern str

Optional regex pattern to validate the cell value.

None
is_description bool

Flag indicating if the value is a description.

False
is_data bool

Flag indicating if the value is a data type.

False
is_url bool

Flag indicating if the value is a URL.

False

Returns:

Type Description
str

The extracted and validated cell value as a string. Returns an empty string if the value is invalid or not provided.

Source code in bam_masterdata/excel/excel_to_entities.py
def extract_value(
    self,
    sheet: "Worksheet",
    row: int,
    column: int,
    validation_pattern: str = None,
    is_description: bool = False,
    is_data: bool = False,
    is_url: bool = False,
) -> str:
    """
    Extracts and validates a value from a specified cell in the Excel sheet.

    Args:
        sheet: The worksheet object.
        row: The row number of the cell (1-based index).
        column: The column number of the cell (1-based index).
        validation_pattern: Optional regex pattern to validate the cell value.
        is_description: Flag indicating if the value is a description.
        is_data: Flag indicating if the value is a data type.
        is_url: Flag indicating if the value is a URL.

    Returns:
        The extracted and validated cell value as a string. Returns an empty string if the value is invalid or not provided.
    """
    value = sheet.cell(row=row, column=column).value

    # No `value` provided
    if not value:
        return ""

    validated = (
        bool(re.match(validation_pattern, str(value)))
        if validation_pattern
        else True
    )
    error_message = f"Invalid value '{value}' at row {row}, column {column} in sheet {sheet.title}"

    if is_description:
        error_message += " Description should follow the schema: English Description + '//' + German Description."
    elif is_data:
        val_upper = str(value).upper()
        allowed_types = [dt.value for dt in DataType]
        is_valid_standard = val_upper in allowed_types
        is_valid_dynamic = False
        if not is_valid_standard and (
            val_upper.startswith("SAMPLE:") or val_upper.startswith("OBJECT:")
        ):
            parts = val_upper.split(":", 1)
            if len(parts) == 2 and parts[1].strip():
                is_valid_dynamic = True
        validated = is_valid_standard or is_valid_dynamic
        error_message += f" The Data Type should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'"
    elif is_url:
        error_message += " It should be an URL or empty"

    if not validated:
        self.logger.error(
            error_message,
            cell_value=value,
            sheet_title=sheet.title,
            row=row,
            column=column,
        )

    return value or ""

get_and_check_property(value, term, coordinate, sheet_title, is_description=False, is_code=False, is_data=False, is_url=False)

Gets a property and checks its format.

Parameters:

Name Type Description Default
value str | bool | None

The string to convert.

required

Returns:

Type Description
str

The property.

Source code in bam_masterdata/excel/excel_to_entities.py
def get_and_check_property(
    self,
    value: str | bool | None,
    term: str,
    coordinate: str,
    sheet_title: str,
    is_description: bool = False,
    is_code: bool = False,
    is_data: bool = False,
    is_url: bool = False,
) -> str:
    """
    Gets a property and checks its format.

    Args:
        value: The string to convert.

    Returns:
        The property.
    """

    # No `value` provided
    if not value:
        return ""

    val = str(value)
    error_message = f"Invalid {term.lower()} value found in the {term} column at position {coordinate} in {sheet_title}."
    if is_description:
        if not re.match(r".*//.*", val):
            self.logger.error(
                error_message
                + "Description should follow the schema: English Description + '//' + German Description. ",
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
    elif is_code:
        if not re.match(r"^\$?[A-Z0-9_.]+$", val):
            self.logger.error(
                error_message,
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
    elif is_data:
        # Normalize data type to uppercase and allow dynamic SAMPLE/OBJECT codes
        val_upper = val.upper()
        allowed_types = [dt.value for dt in DataType]
        is_valid_standard = val_upper in allowed_types
        is_valid_dynamic = False

        if not is_valid_standard and (
            val_upper.startswith("SAMPLE:") or val_upper.startswith("OBJECT:")
        ):
            parts = val_upper.split(":", 1)
            if len(parts) == 2 and parts[1].strip():
                is_valid_dynamic = True

        if not is_valid_standard and not is_valid_dynamic:
            self.logger.error(
                error_message
                + f"The Data Type should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                term=term,
                cell_value=val_upper,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
        val = val_upper
    elif is_url:
        if not re.match(
            r"https?://(?:www\.)?[a-zA-Z0-9-._~:/?#@!$&'()*+,;=%]+", val
        ):
            self.logger.error(
                error_message,
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
    else:
        if not re.match(r".*", val):
            self.logger.error(
                error_message,
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
    return val

get_last_non_empty_row(sheet, start_index)

Finds the last non-empty row before encountering a completely empty row.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
start_index int

The row number to start checking from (1-based index).

required

Returns:

Type Description
int | None

The row number of the last non-empty row before an empty row is encountered,

int | None

or None if no non-empty rows are found starting from the given index.

Source code in bam_masterdata/excel/excel_to_entities.py
def get_last_non_empty_row(
    self, sheet: "Worksheet", start_index: int
) -> int | None:
    """
    Finds the last non-empty row before encountering a completely empty row.

    Args:
        sheet: The worksheet object.
        start_index: The row number to start checking from (1-based index).

    Returns:
        The row number of the last non-empty row before an empty row is encountered,
        or None if no non-empty rows are found starting from the given index.
    """
    if start_index < 1 or start_index > sheet.max_row:
        raise ValueError(
            f"Invalid start index: {start_index}. It must be between 1 and {sheet.max_row}."
        )

    last_non_empty_row = None
    for row in range(start_index, sheet.max_row + 1):
        if all(
            sheet.cell(row=row, column=col).value in (None, "")
            for col in range(1, sheet.max_column + 1)
        ):
            return last_non_empty_row  # Return the last non-empty row before the current empty row

        last_non_empty_row = row  # Update the last non-empty row

    return last_non_empty_row  # If no empty row is encountered, return the last non-empty row

index_to_excel_column(index)

Converts a 1-based index to an Excel column name.

Parameters:

Name Type Description Default
index int

The 1-based index to convert.

required

Returns:

Type Description
str

The corresponding Excel column name.

Source code in bam_masterdata/excel/excel_to_entities.py
def index_to_excel_column(self, index: int) -> str:
    """
    Converts a 1-based index to an Excel column name.

    Args:
        index: The 1-based index to convert.

    Returns:
        The corresponding Excel column name.
    """
    if not index >= 1:
        raise ValueError("Index must be a positive integer starting from 1.")

    column = ""
    while index > 0:
        index, remainder = divmod(index - 1, 26)
        column = chr(65 + remainder) + column
    return column

process_entity(sheet, start_index_row, header_terms, expected_terms, entity_type)

Process an entity type block in the Excel sheet and return its attributes as a dictionary.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
start_index_row int

The row where the current entity type begins (1-based index).

required
header_terms list[str]

List of header terms in the entity block.

required
expected_terms list[str]

List of expected terms to extract from the entity block.

required
entity_type str

The type of the entity (e.g., SAMPLE_TYPE, OBJECT_TYPE).

required

Returns:

Type Description
dict[str, Any]

A dictionary containing the attributes of the entity.

Source code in bam_masterdata/excel/excel_to_entities.py
def process_entity(
    self,
    sheet: "Worksheet",
    start_index_row: int,
    header_terms: list[str],
    expected_terms: list[str],
    entity_type: str,
) -> dict[str, Any]:
    """
    Process an entity type block in the Excel sheet and return its attributes as a dictionary.

    Args:
        sheet: The worksheet object.
        start_index_row: The row where the current entity type begins (1-based index).
        header_terms: List of header terms in the entity block.
        expected_terms: List of expected terms to extract from the entity block.
        entity_type: The type of the entity (e.g., SAMPLE_TYPE, OBJECT_TYPE).

    Returns:
        A dictionary containing the attributes of the entity.
    """
    attributes: dict = {}
    cell_value: Any = ""

    for term in expected_terms:
        if term not in header_terms:
            self.logger.error(f"{term} not found in the headers.", term=term)
        else:
            term_index = header_terms.index(term)
            cell = sheet.cell(row=start_index_row + 2, column=term_index + 1)
            cell_value = self.extract_value(
                sheet,
                start_index_row + 2,
                term_index + 1,
                self.VALIDATION_RULES[entity_type][term].get("pattern"),
            )

            # Handle boolean conversion
            if self.VALIDATION_RULES[entity_type][term].get("is_bool"):
                cell_value = self.str_to_bool(
                    value=cell_value,
                    term=term,
                    coordinate=cell.coordinate,
                    sheet_title=sheet.title,
                )

            # Handle data type validation
            elif self.VALIDATION_RULES[entity_type][term].get("is_data"):
                allowed_types = [dt.value for dt in DataType]
                cell_value_upper = str(cell_value).upper()
                is_valid_standard = cell_value_upper in allowed_types
                is_valid_dynamic = False
                if not is_valid_standard and (
                    cell_value_upper.startswith("SAMPLE:")
                    or cell_value_upper.startswith("OBJECT:")
                ):
                    parts = cell_value_upper.split(":", 1)
                    if len(parts) == 2 and parts[1].strip():
                        is_valid_dynamic = True
                if not is_valid_standard and not is_valid_dynamic:
                    self.logger.error(
                        f"Invalid Data Type: {cell_value} in {cell.coordinate} (Sheet: {sheet.title}). Should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                        term=term,
                        cell_value=cell_value,
                        cell_coordinate=cell.coordinate,
                        sheet_title=sheet.title,
                    )
                else:
                    cell_value = (
                        cell_value_upper
                        if isinstance(cell_value, str)
                        else cell_value
                    )

            # Handle additional validation for "Generated code prefix"
            elif (
                self.VALIDATION_RULES[entity_type][term].get("extra_validation")
                == "is_reduced_version"
            ):
                if not is_reduced_version(cell_value, attributes.get("code", "")):
                    self.logger.warning(
                        f"Invalid {term} value '{cell_value}' in {cell.coordinate} (Sheet: {sheet.title}). "
                        f"Generated code prefix should be part of the 'Code' {attributes.get('code', '')}.",
                        term=term,
                        cell_value=cell_value,
                        cell_coordinate=cell.coordinate,
                        sheet_title=sheet.title,
                    )

            # Handle validation script (allows empty but must match pattern if provided)
            elif (
                self.VALIDATION_RULES[entity_type][term].get("allow_empty")
                and not cell_value
            ):
                cell_value = None

            # Handle URL template validation (allows empty but must be a valid URL)
            elif (
                self.VALIDATION_RULES[entity_type][term].get("is_url")
                and cell_value
            ):
                url_pattern = self.VALIDATION_RULES[entity_type][term].get(
                    "pattern"
                )
                if not re.match(url_pattern, str(cell_value)):
                    self.logger.error(
                        f"Invalid URL format: {cell_value} in {cell.coordinate} (Sheet: {sheet.title})",
                        cell_value=cell_value,
                        cell_coordinate=cell.coordinate,
                        sheet_title=sheet.title,
                    )

            # Add the extracted value to the attributes dictionary
            attributes[self.VALIDATION_RULES[entity_type][term].get("key")] = (
                cell_value
            )

    if self.row_cell_info:
        attributes["row_location"] = f"A{start_index_row}"
    return attributes

process_term(term, cell_value, coordinate, sheet_title)

Processes a term by converting it to a boolean if necessary or checking its validity.

Parameters:

Name Type Description Default
term str

The term being processed.

required
cell_value Any

The value of the cell.

required
coordinate str

The coordinate of the cell in the sheet.

required
sheet_title str

The title of the sheet.

required

Returns:

Type Description
Any

The processed value, either as a boolean or the original value after validation.

Source code in bam_masterdata/excel/excel_to_entities.py
def process_term(
    self, term: str, cell_value: Any, coordinate: str, sheet_title: str
) -> Any:
    """
    Processes a term by converting it to a boolean if necessary or checking its validity.

    Args:
        term: The term being processed.
        cell_value: The value of the cell.
        coordinate: The coordinate of the cell in the sheet.
        sheet_title: The title of the sheet.

    Returns:
        The processed value, either as a boolean or the original value after validation.
    """
    # Check if the term is a boolean type
    if term in ("Mandatory", "Show in edit views"):
        return self.str_to_bool(
            value=cell_value,
            term=term,
            coordinate=coordinate,
            sheet_title=sheet_title,
        )
    # Check and validate the property
    return self.get_and_check_property(
        value=cell_value,
        term=term,
        coordinate=coordinate,
        sheet_title=sheet_title,
        is_code=(term in ["Code", "Vocabulary code"]),
        is_data=(term == "Data type"),
    )

properties_to_dict(sheet, start_index_row, last_non_empty_row)

Extracts properties from an Entity type block in the Excel sheet and returns them as a dictionary.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
start_index_row int

Row where the current entity type begins (1-based index).

required
last_non_empty_row int

Row where the current entity type finish (1-based index).

required

Returns:

Type Description
dict[str, dict[str, Any]]

A dictionary where each key is a property code and the value is a dictionary

dict[str, dict[str, Any]]

containing the attributes of the property.

Source code in bam_masterdata/excel/excel_to_entities.py
def properties_to_dict(
    self, sheet: "Worksheet", start_index_row: int, last_non_empty_row: int
) -> dict[str, dict[str, Any]]:
    """
    Extracts properties from an Entity type block in the Excel sheet and returns them as a dictionary.

    Args:
        sheet: The worksheet object.
        start_index_row: Row where the current entity type begins (1-based index).
        last_non_empty_row: Row where the current entity type finish (1-based index).

    Returns:
        A dictionary where each key is a property code and the value is a dictionary
        containing the attributes of the property.
    """
    property_dict: dict = {}
    expected_terms = [
        "Code",
        "Description",
        "Mandatory",
        "Show in edit views",
        "Section",
        "Property label",
        "Data type",
        "Vocabulary code",
        "Metadata",
        "Dynamic script",
        # ! these are not used
        # "Unique",
        # "Internal assignment",
    ]

    # Determine the header row index
    header_index = start_index_row + 3
    row_headers = [(cell.value, cell.coordinate) for cell in sheet[header_index]]
    # And store how many properties are for the entity
    n_properties = last_non_empty_row - header_index
    if n_properties < 0:
        self.logger.error(
            f"No properties found for the entity in sheet {sheet.title} starting at row {start_index_row}."
        )
        return property_dict

    # Initialize a dictionary to store extracted columns
    extracted_columns: dict[str, list] = {term: [] for term in expected_terms}
    if self.row_cell_info:
        extracted_columns["row_location"] = []

    # Extract columns for each expected term
    for i, (term, coordinate) in enumerate(row_headers):
        if term not in expected_terms:
            log_func = (
                self.logger.warning
                if term
                in (
                    "Mandatory",
                    "Show in edit views",
                    "Section",
                    "Metadata",
                    "Dynamic script",
                    "Vocabulary code",
                    # ! these are not used
                    # "Unique",
                    # "Internal assignment",
                )
                else self.logger.error
            )
            log_func(f"'{term}' not found in the properties headers.", term=term)
            continue

        # Excel column letter from the coordinate
        term_letter = coordinate[0]

        # Extract values from the column
        for cell_property in sheet[term_letter][header_index:last_non_empty_row]:
            extracted_columns[term].append(
                self.process_term(
                    term, cell_property.value, cell_property.coordinate, sheet.title
                )
            )
            if self.row_cell_info:
                extracted_columns["row_location"].append(cell_property.coordinate)

    # Combine extracted values into a dictionary
    for i in range(n_properties):
        code = extracted_columns.get("Code", [])
        if not code:
            self.logger.error(
                f"'Code' not found in the properties headers for sheet {sheet.title}."
            )
            return property_dict
        code = code[i]
        property_dict[code] = {"permId": code, "code": code}
        for key, pybis_val in {
            "Description": "description",
            "Section": "section",
            "Mandatory": "mandatory",
            "Show in edit views": "show_in_edit_views",
            "Property label": "label",
            "Data type": "dataType",
            "Vocabulary code": "vocabularyCode",
        }.items():
            data_column = extracted_columns.get(key, [])
            if not data_column:
                continue
            cell_value = data_column[i]
            if key == "Data type":
                object_code = None
                normalized_value = (
                    str(cell_value).upper()
                    if isinstance(cell_value, str)
                    else cell_value
                )
                if isinstance(normalized_value, str) and ":" in normalized_value:
                    prefix, dynamic_code = normalized_value.split(":", 1)
                    if prefix in ("SAMPLE", "OBJECT") and dynamic_code.strip():
                        object_code = dynamic_code.strip()
                        normalized_value = DataType.OBJECT.value
                property_dict[code][pybis_val] = normalized_value
                if object_code:
                    property_dict[code]["objectCode"] = object_code
            else:
                property_dict[code][pybis_val] = cell_value
        if self.row_cell_info:
            property_dict[code]["row_location"] = (
                extracted_columns.get("row_location")[i],
            )
        # Only add optional fields if they exist in extracted_columns
        optional_fields = [
            "Metadata",
            "Dynamic script",
            "Unique",
            "Internal assignment",
        ]
        for field in optional_fields:
            if (
                field in extracted_columns
            ):  # Check if the field exists in the extracted columns
                if extracted_columns[field][i] == "":
                    extracted_columns[field][i] = None
                property_dict[extracted_columns["Code"][i]][
                    field.lower().replace(" ", "_")
                ] = extracted_columns[field][i]

    return property_dict

str_to_bool(value, term, coordinate, sheet_title)

Converts a string to a boolean value.

Parameters:

Name Type Description Default
value str | bool | None

The string to convert.

required

Returns:

Type Description
bool

The boolean value.

Source code in bam_masterdata/excel/excel_to_entities.py
def str_to_bool(
    self,
    value: str | bool | None,
    term: str,
    coordinate: str,
    sheet_title: str,
) -> bool:
    """
    Converts a string to a boolean value.

    Args:
        value: The string to convert.

    Returns:
        The boolean value.
    """
    # No `value` provided
    if not value:
        return False

    val = str(value).strip().lower()
    if val not in ["true", "false"]:
        self.logger.error(
            f"Invalid {term.lower()} value found in the {term} column at position {coordinate} in {sheet_title}. Accepted values: TRUE or FALSE.",
            term=term,
            cell_value=val,
            cell_coordinate=coordinate,
            sheet_title=sheet_title,
        )
    return val == "true"

terms_to_dict(sheet, start_index_row, last_non_empty_row)

Extracts terms from a Vocabulary block in the Excel sheet and returns them as a dictionary.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
start_index_row int

Row where the current entity type begins (1-based index).

required
last_non_empty_row int

Row where the current entity type finish (1-based index).

required

Returns:

Type Description
dict[str, dict[str, Any]]

A dictionary where each key is a vocabulary term code and the value is a dictionary

dict[str, dict[str, Any]]

containing the attributes of the vocabulary term.

Source code in bam_masterdata/excel/excel_to_entities.py
def terms_to_dict(
    self, sheet: "Worksheet", start_index_row: int, last_non_empty_row: int
) -> dict[str, dict[str, Any]]:
    """
    Extracts terms from a Vocabulary block in the Excel sheet and returns them as a dictionary.

    Args:
        sheet: The worksheet object.
        start_index_row: Row where the current entity type begins (1-based index).
        last_non_empty_row: Row where the current entity type finish (1-based index).

    Returns:
        A dictionary where each key is a vocabulary term code and the value is a dictionary
        containing the attributes of the vocabulary term.
    """
    terms_dict = {}
    expected_terms = ["Code", "Description", "Url template", "Label", "Official"]

    header_index = start_index_row + 3
    row_headers = [cell.value for cell in sheet[header_index]]

    # Initialize a dictionary to store extracted columns
    extracted_columns: dict[str, list] = {term: [] for term in expected_terms}

    # Helper function to process each term
    def process_term_cell(term, cell_value, coordinate, sheet_title):
        if term == "Official":
            return self.str_to_bool(
                value=cell_value,
                term=term,
                coordinate=coordinate,
                sheet_title=sheet_title,
            )
        return self.get_and_check_property(
            value=cell_value,
            term=term,
            coordinate=coordinate,
            sheet_title=sheet_title,
            is_code=(term == "Code"),
            is_url=(term == "Url template"),
        )

    # Extract columns for each expected term
    for term in expected_terms:
        if term not in row_headers:
            self.logger.warning(
                f"{term} not found in the properties headers.", term=term
            )
            continue

        # Get column index and Excel letter
        term_index = row_headers.index(term) + 1
        term_letter = self.index_to_excel_column(term_index)

        # Extract values from the column
        for cell in sheet[term_letter][header_index:last_non_empty_row]:
            extracted_columns[term].append(
                process_term_cell(term, cell.value, cell.coordinate, sheet.title)
            )

    if not extracted_columns.get("Code"):
        self.logger.error(
            f"The required 'Code' column for terms was not found in sheet {sheet.title}."
        )
        return {}

    # Combine extracted values into a dictionary safely
    for i in range(len(extracted_columns["Code"])):
        code = extracted_columns["Code"][i]
        terms_dict[code] = {
            "permId": code,
            "code": code,
        }
        for key, pybis_val in {
            "Description": "descriptions",
            "Url template": "url_template",
            "Label": "label",
            "Official": "official",
        }.items():
            values = extracted_columns.get(key, [])
            if len(values) > i:
                terms_dict[code][pybis_val] = values[i]

    return terms_dict

bam_masterdata.cli.entities_to_excel

entities_to_excel(worksheet, module_path, definitions_module)

Export entities to the Excel file. The Python modules are imported using the function import_module, and their contents are inspected (using inspect) to find the classes in the datamodel containing defs and with a model_to_json method defined. Each row is then appended to the worksheet.

Parameters:

Name Type Description Default
worksheet Worksheet

The worksheet to append the entities.

required
module_path str

Path to the Python module file.

required
definitions_module Any

The module containing the definitions of the entities. This is used to match the header definitions of the entities.

required
Source code in bam_masterdata/cli/entities_to_excel.py
def entities_to_excel(
    worksheet: "Worksheet",
    module_path: str,
    definitions_module: Any,
) -> None:
    """
    Export entities to the Excel file. The Python modules are imported using the function `import_module`,
    and their contents are inspected (using `inspect`) to find the classes in the datamodel containing
    `defs` and with a `model_to_json` method defined. Each row is then appended to the `worksheet`.

    Args:
        worksheet (Worksheet): The worksheet to append the entities.
        module_path (str): Path to the Python module file.
        definitions_module (Any): The module containing the definitions of the entities. This is used
            to match the header definitions of the entities.
    """
    def_members = inspect.getmembers(definitions_module, inspect.isclass)
    module = import_module(module_path=module_path)

    # Inspect Python modules and their objects and print them to Excel
    for _, obj in inspect.getmembers(module, inspect.isclass):
        # Ensure the class has the `model_to_json` method
        if not hasattr(obj, "defs") or not callable(getattr(obj, "model_to_json")):
            continue

        obj_instance = obj()

        # Entity title
        obj_definitions = obj_instance.defs
        worksheet.append([obj_definitions.excel_name])

        # Entity header definitions and values
        for def_name, def_cls in def_members:
            if def_name == obj_definitions.name:
                break
        # Appending headers and values in worksheet
        excel_headers = []
        header_values = []
        for field, excel_header in obj_definitions.excel_headers_map.items():
            header_values.append(getattr(obj_definitions, field))
            excel_headers.append(excel_header)
        worksheet.append(excel_headers)
        worksheet.append(header_values)

        # Properties assignment for ObjectType, DatasetType, and CollectionType
        if obj_instance.base_name in ["ObjectType", "DatasetType", "CollectionType"]:
            if not obj_instance.properties:
                continue
            worksheet.append(
                list(obj_instance.properties[0].excel_headers_map.values())
            )
            for prop in obj_instance.properties:
                row = []
                for field in prop.excel_headers_map.keys():
                    if field == "data_type":
                        if prop.data_type == DataType.OBJECT and getattr(
                            prop, "object_code", None
                        ):
                            val = f"SAMPLE:{prop.object_code}"
                        else:
                            val = prop.data_type.value
                    else:
                        val = getattr(prop, field)
                    row.append(val)
            worksheet.append(row)
        # Terms assignment for VocabularyType
        elif obj_instance.base_name == "VocabularyType":
            if not obj_instance.terms:
                continue
            worksheet.append(list(obj_instance.terms[0].excel_headers_map.values()))
            for term in obj_instance.terms:
                worksheet.append(
                    getattr(term, f_set) for f_set in term.excel_headers_map.keys()
                )
        worksheet.append([""])  # empty row after entity definitions

bam_masterdata.cli.entities_to_rdf

entities_to_rdf(graph, module_path, logger)

Convert the entities defined in the specified module to RDF triples and add them to the graph. The function uses the model_to_rdf method defined in each class to convert the class attributes to RDF triples. The function also adds the PropertyType and other entity types as placeholders in the graph.

Parameters:

Name Type Description Default
graph Graph

The RDF graph to which the entities are added.

required
module_path str

The path to the module containing the entities to be converted.

required
logger BoundLoggerLazyProxy

The logger to log messages.

required
Source code in bam_masterdata/cli/entities_to_rdf.py
def entities_to_rdf(
    graph: "Graph", module_path: str, logger: "BoundLoggerLazyProxy"
) -> None:
    """
    Convert the entities defined in the specified module to RDF triples and add them to the graph. The function
    uses the `model_to_rdf` method defined in each class to convert the class attributes to RDF triples. The
    function also adds the PropertyType and other entity types as placeholders in the graph.

    Args:
        graph (Graph): The RDF graph to which the entities are added.
        module_path (str): The path to the module containing the entities to be converted.
        logger (BoundLoggerLazyProxy): The logger to log messages.
    """
    rdf_graph_init(graph)

    module = import_module(module_path=module_path)

    # All datamodel modules
    # OBJECT/DATASET/COLLECTION TYPES
    # skos:prefLabel used for class names
    # skos:definition used for `description` (en, de)
    # dc:identifier used for `code`  # ! only defined for internal codes with $ symbol
    # parents defined from `code`
    # assigned properties can be Mandatory or Optional, can be PropertyType or ObjectType
    # ? For OBJECT TYPES
    # ? `generated_code_prefix`, `auto_generate_codes`?
    for name, obj in inspect.getmembers(module, inspect.isclass):
        # Ensure the class has the `model_to_rdf` method
        if not hasattr(obj, "defs") or not callable(getattr(obj, "model_to_rdf")):
            continue
        try:
            # Instantiate the class and call the method
            entity = obj()
            entity.model_to_rdf(namespace=BAM, graph=graph, logger=logger)
        except Exception as err:
            click.echo(f"Failed to process class {name} in {module_path}: {err}")

rdf_graph_init(g)

Initialize the RDF graph with base namespaces, annotation properties, and internal BAM properties. This function also creates placeholders for PropertyType and other entity types. The graph is to be printed out in RDF/XML format in the entities_to_rdf function.

Parameters:

Name Type Description Default
g Graph

The RDF graph to be initialized.

required
Source code in bam_masterdata/cli/entities_to_rdf.py
def rdf_graph_init(g: "Graph") -> None:
    """
    Initialize the RDF graph with base namespaces, annotation properties, and internal BAM properties. This
    function also creates placeholders for PropertyType and other entity types. The graph is to be printed out
    in RDF/XML format in the `entities_to_rdf` function.

    Args:
        g (Graph): The RDF graph to be initialized.
    """
    # Adding base namespaces
    g.bind("dc", DC)
    g.bind("owl", OWL)
    g.bind("rdf", RDF)
    g.bind("rdfs", RDFS)
    g.bind("bam", BAM)
    g.bind("prov", PROV)

    # Adding annotation properties from base namespaces
    annotation_props = [
        RDFS.label,
        RDFS.comment,
        DC.identifier,
    ]
    for prop in annotation_props:
        g.add((prop, RDF.type, OWL.AnnotationProperty))

    # Custom annotation properties from openBIS: `dataType`, `propertyLabel
    custom_annotation_props = {
        BAM[
            "dataType"
        ]: """Represents the data type of a property as defined in the openBIS platform.
        This annotation is used to ensure alignment with the native data types in openBIS,
        facilitating seamless integration and data exchange.

        The allowed values for this annotation correspond directly to the openBIS type system,
        including BOOLEAN, CONTROLLEDVOCABULARY, DATE, HYPERLINK, INTEGER, MULTILINE_VARCHAR, OBJECT,
        REAL, TIMESTAMP, VARCHAR, and XML.

        While `bam:dataType` is primarily intended for internal usage with openBIS, mappings to
        standard vocabularies such as `xsd` (e.g., `xsd:boolean`, `xsd:string`) are possible to use and documented to
        enhance external interoperability. The full mapping is:
        - BOOLEAN: xsd:boolean
        - CONTROLLEDVOCABULARY: xsd:string
        - DATE: xsd:date
        - HYPERLINK: xsd:anyURI
        - INTEGER: xsd:integer
        - MULTILINE_VARCHAR: xsd:string
        - OBJECT: bam:ObjectType
        - REAL: xsd:decimal
        - TIMESTAMP: xsd:dateTime
        - VARCHAR: xsd:string
        - XML: xsd:string""",
        BAM[
            "propertyLabel"
        ]: """A UI-specific annotation used in openBIS to provide an alternative label for a property
        displayed in the frontend. Not intended for semantic reasoning or interoperability beyond openBIS.""",
    }
    for custom_prop, custom_prop_def in custom_annotation_props.items():
        g.add((custom_prop, RDF.type, OWL.AnnotationProperty))
        g.add(
            (
                custom_prop,
                RDFS.label,
                Literal(f"bam:{custom_prop.split('/')[-1]}", lang="en"),
            )
        )
        g.add((custom_prop, RDFS.comment, Literal(custom_prop_def, lang="en")))

    # Internal BAM properties
    # ? `section`, `ordinal`, `show_in_edit_views`?
    bam_props_uri = {
        BAM["hasMandatoryProperty"]: [
            (RDF.type, OWL.ObjectProperty),
            # (RDFS.domain, OWL.Class),
            (RDFS.range, BAM.PropertyType),
            (RDFS.label, Literal("hasMandatoryProperty", lang="en")),
            (
                RDFS.comment,
                Literal(
                    "The property must be mandatorily filled when creating the object in openBIS.",
                    lang="en",
                ),
            ),
        ],
        BAM["hasOptionalProperty"]: [
            (RDF.type, OWL.ObjectProperty),
            # (RDFS.domain, OWL.Class),
            (RDFS.range, BAM.PropertyType),
            (RDFS.label, Literal("hasOptionalProperty", lang="en")),
            (
                RDFS.comment,
                Literal(
                    "The property is optionally filled when creating the object in openBIS.",
                    lang="en",
                ),
            ),
        ],
        BAM["referenceTo"]: [
            (RDF.type, OWL.ObjectProperty),
            (RDFS.domain, BAM.PropertyType),  # Restricting domain to PropertyType
            # (RDFS.range, OWL.Class),  # Explicitly setting range to ObjectType
            (RDFS.label, Literal("referenceTo", lang="en")),
            (
                RDFS.comment,
                Literal(
                    "The property is referencing an object existing in openBIS.",
                    lang="en",
                ),
            ),
        ],
    }
    for prop_uri, obj_properties in bam_props_uri.items():
        for prop in obj_properties:  # type: ignore
            g.add((prop_uri, prop[0], prop[1]))  # type: ignore

    # Adding base PropertyType and other objects as placeholders
    # ! add only PropertyType
    prop_type_description = """A conceptual placeholder used to define and organize properties as first-class entities.
        PropertyType is used to place properties and define their metadata, separating properties from the
        entities they describe.

        In integration scenarios:
        - PropertyType can align with `BFO:Quality` for inherent attributes.
        - PropertyType can represent `BFO:Role` if properties serve functional purposes.
        - PropertyType can be treated as a `prov:Entity` when properties participate in provenance relationships."""
    for entity in ["PropertyType", "ObjectType", "CollectionType", "DatasetType"]:
        entity_uri = BAM[entity]
        g.add((entity_uri, RDF.type, OWL.Thing))
        g.add((entity_uri, RDFS.label, Literal(entity, lang="en")))
        if entity == "PropertyType":
            g.add((entity_uri, RDFS.comment, Literal(prop_type_description, lang="en")))

bam_masterdata.cli.run_parser

ResolvedDestination dataclass

Runtime openBIS destination for an object.

Unlike Destination in bam_masterdata/metadata/destination.py, this contains resolved pybis entities and is used internally by RunParsers only.

Source code in bam_masterdata/cli/run_parser.py
@dataclass(frozen=True)
class ResolvedDestination:
    """
    Runtime openBIS destination for an object.

    Unlike `Destination` in `bam_masterdata/metadata/destination.py`, this contains resolved pybis entities and is used
    internally by `RunParsers` only.
    """

    space: Space
    project: Project
    collection: Experiment | None = None

RunParsers

Run configured parsers and persist the resulting data model to openBIS.

The class parses input files into an internal collection of objects and relationships, creates or updates the corresponding objects in openBIS, attaches datasets, and creates parent-child relationships between objects.

Objects can be stored directly under a project or inside a collection, depending on the configured collection_name.

Source code in bam_masterdata/cli/run_parser.py
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
class RunParsers:
    """
    Run configured parsers and persist the resulting data model to openBIS.

    The class parses input files into an internal collection of objects and relationships, creates
    or updates the corresponding objects in openBIS, attaches datasets, and creates parent-child
    relationships between objects.

    Objects can be stored directly under a project or inside a collection, depending on the
    configured `collection_name`.
    """

    def __init__(
        self,
        openbis: Openbis | None = None,
        space_name: str = "",
        project_name: str = "PROJECT",
        collection_name: str = "",
        files_parser: dict[AbstractParser, list[str]] | None = None,
        logger: BoundLoggerLazyProxy = logger,
        collection_type: str = "COLLECTION",
    ):
        # Initial checks
        if openbis is None:
            raise ValueError("An openBIS instance must be provided.")
        if not project_name:
            raise ValueError("The project_name must be specified.")

        # Ensure the files_parser is not empty
        if not files_parser:
            raise ValueError(
                "At least one valid parser with its associated files must be provided."
            )

        # Ensure collection_type is valid
        if collection_type not in {"COLLECTION", "DEFAULT_EXPERIMENT"}:
            raise ValueError(
                f"Invalid collection_type '{collection_type}'. "
                "Must be either 'COLLECTION' or 'DEFAULT_EXPERIMENT'."
            )

        self.logger = logger
        self.openbis = openbis
        self.collection_name = collection_name
        self.files_parser = files_parser
        # initializing space/project/collection from openBIS
        self._openbis_init(
            space_name=space_name,
            project_name=project_name,
            collection_name=collection_name,
            collection_type=collection_type,
        )

    def _openbis_init(
        self,
        space_name: str,
        project_name: str,
        collection_name: str,
        collection_type: str = "COLLECTION",
    ):
        """
        Initializes the Space, Project, and (optionally) Experiment objects from openBIS based
        on the input names and collection type. It also creates an instance of `CollectionType`.

        Args:
            space_name (str): The name of the space in openBIS.
            project_name (str): The name of the project to create or get.
            collection_name (str): The name of the collection to create or get.
            collection_type (str): The type of the collection. Defaults to "COLLECTION".
        """
        self.space = self._get_space(space_name)
        self.project = self._get_project(project_name)
        self.collection_openbis = self._get_collection(collection_name, collection_type)
        self.collection = CollectionType()

    def _get_space(self, space_name: str = "") -> Space:
        """
        Gets the OpenBis space from the specified `space_name`. If no space name is provided, it attempts to find a default space for the username.

        Args:
            space_name (str): The name of the space in openBIS. Defaults to "".

        Returns:
            (Space): The retrieved space.
        """
        try:
            return self.openbis.get_space(space_name)
        except Exception:
            username = self.openbis.username.upper()

            for openbis_space in self.openbis.get_spaces():
                if openbis_space.code.endswith(username):
                    self.logger.warning(
                        f"Space '{space_name}' not found. Using default space: {openbis_space.code}"
                    )
                    return openbis_space

        raise ValueError(
            "No space found for the specified name or default username. Please provide a valid space name."
        )

    def _get_project(self, project_name: str) -> Project:
        """
        Gets project if `project_name` exists in the space in openBIS. Otherwise, creates a new project with the specified name.

        Args:
            project_name (str): The name of the project to create or get.

        Returns:
            (Project): The created or retrieved project.
        """
        try:
            return self.space.get_project(project_name)
        except Exception:
            self.logger.info(
                f"Project '{project_name}' not found in the space '{self.space.code}' in openBIS. Creating a new project."
            )
            project_name = project_name.replace(" ", "_").upper()
            project = self.space.new_project(
                code=project_name,
                description=f"New project named {project_name} created via automated parsing with `bam_masterdata`.",
            )
            project.save()
            return project

    def _get_collection(
        self,
        collection_name: str,
        collection_type: str = "COLLECTION",
    ) -> Experiment | Project:
        """
        Gets or creates an openBIS collection. If no collection name is provided, objects are
        attached directly to the project.

        Args:
            collection_name (str): The name of the collection to create or retrieve.
            collection_type (str): openBIS collection type. Defaults to "COLLECTION".

        Returns:
            (Experiment | Project): The created or retrieved collection, or the project if no `collection_name` is provided.
        """
        if not collection_name:
            self.logger.info(
                "No collection name specified. Attaching objects directly to the project."
            )
            return self.project

        collection_code = collection_name.replace(" ", "_").upper()

        existing_collections = {c.code: c for c in self.project.get_collections()}

        if collection_code in existing_collections:
            self.logger.info(
                f"Collection '{collection_code}' already exists. Using the existing collection."
            )
            return existing_collections[collection_code]

        self.logger.info(
            f"Collection '{collection_code}' not found. Creating a new collection of type '{collection_type}'."
        )
        collection = self.openbis.new_collection(
            code=collection_code,
            type=collection_type,
            project=self.project,
        )
        collection.save()
        return collection

    def _get_existing_space(self, space_name: str) -> Space:
        """
        Retrieve an existing openBIS space without fallback or creation. This is used for parser-defined
        destinations.

        Args:
            space_name (str): The name of the space in openBIS.

        Returns:
            Space: The retrieved space.
        """
        try:
            return self.openbis.get_space(space_name)
        except Exception:
            raise ValueError(f"Destination space '{space_name}' does not exist.")

    def _get_existing_project(self, space: Space, project_name: str) -> Project:
        """
        Retrieve an existing project from `space` without creating it.

        Args:
            space (Space): The space where the project resides.
            project_name (str): The name of the project to create or get.

        Returns:
            (Project): The retrieved project.
        """
        try:
            return space.get_project(project_name)
        except Exception:
            raise ValueError(
                f"Destination project '{project_name}' does not exist in space '{space.code}'."
            )

    def _get_existing_collection(
        self,
        project: Project,
        collection_name: str,
    ) -> Experiment:
        """
        Retrieve an existing collection from `project` without creating it.

        Args:
            project (Project): The project where the collection resides.
            collection_name (str): The name of the collection to create or retrieve.

        Returns:
            (Experiment | Project): The retrieved collection.
        """
        existing_collections = {
            collection.code: collection for collection in project.get_collections()
        }

        try:
            return existing_collections[collection_name]
        except KeyError:
            raise ValueError(
                f"Destination collection '{collection_name}' does not exist in project '{project.code}'."
            )

    def _resolve_destination(self, object_id: str) -> ResolvedDestination:
        """
        Resolve the openBIS destination for an attached object.

        Objects without an explicit Destination use the complete default RunParsers destination.

        An explicit Destination overrides object placement. Missing space or project values inherit
        from the runner, while an omitted collection means that the object is stored directly under
        the resolved project.

        Args:
            object_id (str): The object id of an attached object during parsing.

        Returns:
            (ResolvedDestination): The resolved destination for the object
        """
        destination = self.collection.destinations.get(object_id)

        # No override: preserve the existing runner destination.
        if destination is None:
            return ResolvedDestination(
                space=self.space,
                project=self.project,
                collection=(self.collection_openbis if self.collection_name else None),
            )

        # Resolve space.
        if destination.space is None:
            space = self.space
        else:
            space = self._get_existing_space(destination.space)

        # Resolve project.
        if destination.project is None:
            if space is self.space:
                project = self.project
            else:
                project = self._get_existing_project(space, self.project.code)
        else:
            project = self._get_existing_project(space, destination.project)

        # An explicit Destination without a collection means project-level storage.
        collection = None
        if destination.collection is not None:
            collection = self._get_existing_collection(project, destination.collection)

        return ResolvedDestination(
            space=space,
            project=project,
            collection=collection,
        )

    def parsing(self) -> None:
        """
        Runs the parser specific class for each of the files specified in `self.files_parser` and adds them to the collection.
        """
        for parser, files in self.files_parser.items():
            parser.parse(files, self.collection, logger=self.logger)

    def _content_hash(self, object_instance: ObjectType, length: int = 16) -> str:
        """
        Generates a hash of the content of the given object instance. This is used to create a unique
        identifier for objects that do not have a specified `code`.

        Args:
            object_instance (ObjectType): The object instance for which to generate a content hash.
            length (int, optional): The length of the hash to generate. Defaults to 16.

        Returns:
            str: The generated content hash.
        """
        serialized_payload = object_instance.to_json()
        return hashlib.sha256(serialized_payload.encode("utf-8")).hexdigest()[:length]

    def _identifier_from_code(self, code: str, destination: ResolvedDestination) -> str:
        """
        Generate an openBIS identifier for `code` at the given destination.

        If no destination is provided, the default RunParsers destination is used. This preserves
        the existing behavior for callers that have not yet been migrated to per-object destinations.

        Args:
            code (str): The code for which to generate an identifier.
            destination (ResolvedDestination): Resolved destination for the object.

        Returns:
            str: The generated identifier.
        """
        if destination.collection is None:
            return f"/{destination.space.code}/{destination.project.code}/{code}"
        return f"/{destination.space.code}/{destination.project.code}/{destination.collection.code}/{code}"

    def _identifier(
        self,
        object_instance: ObjectType,
        destination: ResolvedDestination,
    ) -> str:
        """
        Generates a unique identifier for the given object instance in openBIS. If the object has a specified `code`,
        it uses that code. If not, it generates a code by combining the `generated_code_prefix` and a hash of the object's content.

        Args:
            object_instance (ObjectType): The object instance for which to generate an identifier.
            destination (ResolvedDestination): Resolved destination for the object.

        Returns:
            identifier (str): The unique identifier for the object in openBIS, ensuring no duplicates.
        """
        code = object_instance.code

        if not code:
            self.logger.info(
                "Object has no specified `code`. Generating one from "
                "`generated_code_prefix` and the hashed content."
            )
            prefix = object_instance.defs.generated_code_prefix
            hash_suffix = self._content_hash(object_instance)
            code = f"{prefix}_{hash_suffix}"
            object_instance.code = code  # Update the code in the object instance

        return self._identifier_from_code(code, destination)

    def _resolve_object_reference(
        self,
        property_name: str,
        value: str | ObjectType,
    ) -> str | None:
        """
        Resolves an OBJECT type property reference to an openBIS identifier.

        Args:
            property_name (str): The name of the property being resolved.
            value (str | ObjectType): The value of the property, which can be a string path or an ObjectType instance.

        Returns:
            str | None: The resolved openBIS identifier, or None if it cannot be resolved.
        """
        # If `value` is a string, verify this exists in openBIS and return the identifier of the object
        if isinstance(value, str):
            try:
                referenced_object = self.openbis.get_object(value)
                return referenced_object.identifier
            except Exception as exc:
                self.logger.error(
                    f"Failed to resolve OBJECT reference '{value}' "
                    f"for property '{property_name}': {exc}"
                )
                return None

        # If `value` is an ObjectType instance, we need to construct the identifier path
        if isinstance(value, ObjectType):
            if not value.code:
                self.logger.warning(
                    f"OBJECT reference for property '{property_name}' "
                    "has no code, skipping."
                )
                return None

            # Try to find this object in `self.openbis_id_map` first (if it's being created in the same batch)
            referenced_identifier = None
            for obj_id, obj_inst in self.collection.attached_objects.items():
                if obj_inst is value and obj_id in self.openbis_id_map:
                    referenced_identifier = self.openbis_id_map[obj_id]
                    break

            # If not, construct identifier from the object's code
            # Assume it's in the same space/project as the current object
            if not referenced_identifier:
                # this is a special case where the object is anyway in the selected RunParsers destination
                default_destination = (
                    ResolvedDestination(
                        space=self.space,
                        project=self.project,
                        collection=(
                            self.collection_openbis if self.collection_name else None
                        ),
                    ),
                )
                referenced_identifier = self._identifier_from_code(
                    value.code, default_destination
                )

            return referenced_identifier

        self.logger.warning(
            f"Unexpected type for OBJECT property '{property_name}': "
            f"{type(value).__name__}"
        )
        return None

    def _load_openbis_props(self, object_instance: ObjectType) -> dict:
        """
        Loads the properties of an object instance into a dictionary suitable for openBIS, where the keys are the
        property codes and the values are the corresponding property values. It also resolves OBJECT type properties to their openBIS identifiers.

        Args:
            object_instance (ObjectType): The object instance containing properties.

        Returns:
            dict: A dictionary of property codes and their corresponding values, ready for openBIS.
        """
        obj_props = {}

        for key, property_metadata in object_instance._property_metadata.items():
            value = getattr(object_instance, key, None)
            if isinstance(value, PropertyTypeAssignment):
                continue

            prop_name = property_metadata.code.lower()

            # For the special case of OBJECT type properties, we need to resolve the reference to an openBIS identifier.
            if property_metadata.data_type == "OBJECT":
                value = self._resolve_object_reference(key, value)
                if value is None:
                    self.logger.warning(
                        f"Skipping OBJECT property '{key}' for object '{object_instance.code}' because it could not be resolved."
                    )
                    continue

            obj_props[prop_name] = value
        return obj_props

    def _save_datasets(self, object_instance: ObjectType, identifier: str) -> None:
        """
        Saves attached datasets in the object instance to the object specified in the `identifier`.

        Args:
            object_instance (ObjectType): The object instance containing datasets to be saved.
            identifier (str): Identifier of the object in openBIS to which the dataset should be attached.
        """
        if len(object_instance.datasets) == 0:
            self.logger.info(
                f"No datasets attached to object '{object_instance.code}' with identifier '{identifier}'."
            )
            return None

        try:
            attached_datasets = self.openbis.new_dataset(
                type="RAW_DATA",
                sample=identifier,
                files=object_instance.datasets,
            )
            attached_datasets.save()
            self.logger.info(
                f"Dataset for files {attached_datasets.files} saved successfully"
            )
        except Exception as e:
            self.logger.warning(
                f"Error saving dataset for files {attached_datasets.files}: {e}"
            )

    def _attach_datasets(self) -> None:
        """
        Attaches files to the collection/project in openBIS for each parser and its associated files.
        """
        for files in self.files_parser.values():
            try:
                if not self.collection_name:
                    # ! This won't work on a project -> datasets only attached to collections in pyBIS
                    self.logger.error(
                        "Cannot attach datasets directly to a project. If you want to store them during this process, specify a collection name."
                    )
                    continue
                else:
                    dataset = self.openbis.new_dataset(
                        type="RAW_DATA",
                        files=files,
                        collection=self.collection_openbis,
                    )
                dataset.save()
            except Exception as e:
                self.logger.warning(f"Error uploading files {files} to openBIS: {e}")
                continue
            self.logger.info(
                f"Files uploaded to openBIS collection {self.collection_openbis.code}."
            )

    def _get_openbis_obj(
        self,
        obj: str | dict,
        object_role: str,
    ) -> Sample | None:
        """
        Gets the openBIS object based on the provided identifier or dictionary. If the object is not found, it logs a warning.

        Args:
            obj: Either a bam-masterdata object ID or a dictionary that can be used to resolve the object using pyBIS.
            object_role: Used for logging (e.g. "parent" or "child").

        Returns:
            (Sample | None): The openBIS object if found, otherwise None.
        """
        # If `obj` is a dictionary, try to extract the identifier from it.
        if isinstance(obj, dict):
            identifier = next(
                (
                    value
                    for key, value in obj.items()
                    if key in {"permId", "code", "identifier"}
                ),
                None,
            )

            if identifier is None:
                self.logger.warning(
                    f"No valid identifier found for {object_role} object: {obj}"
                )
                return None
        # If it's a string, look it up in the `openbis_id_map`.
        else:
            identifier = self.openbis_id_map.get(obj)
            if identifier is None:
                self.logger.warning(
                    f"No openBIS identifier found for {object_role} object '{obj}'."
                )
                return None

        # Try to get identifier from an object existing in openBIS
        try:
            return self.openbis.get_object(identifier)
        except Exception as exc:
            self.logger.warning(
                f"Failed to fetch {object_role} object '{identifier}': {exc}"
            )
            return None

    def _load_relationship_objs(
        self, parent: str | dict, child: str | dict
    ) -> tuple[Sample | None, Sample | None]:
        """
        Loads the parent and child objects for a relationship based on the provided identifiers or dictionaries.
        If either object is not found, it logs a warning.

        Args:
            parent (str | dict): The parent object, which can be a bam-masterdata object ID or a dictionary that can be used to resolve the object using pyBIS.
            child (str | dict): The child object, which can be a bam-masterdata object ID or a dictionary that can be used to resolve the object using pyBIS.

        Returns:
            (tuple[Sample | None, Sample | None]): A tuple containing the parent and child openBIS objects. If either object is not found, it will be (None, None).
        """
        parent_obj = self._get_openbis_obj(parent, "parent")
        child_obj = self._get_openbis_obj(child, "child")

        if parent_obj is None or child_obj is None:
            return None, None
        return parent_obj, child_obj

    def run(self) -> None:
        """
        Runs the complete parsing and openBIS import workflow.

        Parses all configured files into the internal collection, creates or updates the corresponding
        objects in openBIS, stores their local-to-openBIS identifier mapping, attaches object-level
        and collection-level datasets, and finally creates the parent-child relationships defined
        during parsing.

        Existing openBIS objects are updated rather than recreated when their identifier already exists.
        """

        # Runs the parser specific class for each of the files specified in `self.files_parser` and adds them to the `self.collection`.
        self.parsing()

        # Storing objects in openBIS and creating a mapping of local IDs to openBIS identifiers for later use in relationships.
        self.openbis_id_map = {}
        for object_id, object_instance in self.collection.attached_objects.items():
            resolved_destination = self._resolve_destination(object_id)
            identifier = self._identifier(object_instance, resolved_destination)
            obj_props = self._load_openbis_props(object_instance)

            try:
                object_openbis = self.openbis.get_object(identifier)
                self.logger.info(
                    f"Object '{identifier}' already exists in openBIS, updating properties."
                )
                object_openbis.set_props(obj_props)  # update properties
            except Exception:
                self.logger.info(f"Creating new object '{object_instance.code}'.")
                object_openbis = self.openbis.new_object(
                    type=object_instance.defs.code,
                    code=object_instance.code,
                    space=resolved_destination.space,
                    project=resolved_destination.project,
                    collection=(
                        resolved_destination.collection
                        if self.collection_name
                        else None
                    ),
                    props=obj_props,
                )
            object_openbis.save()

            # save local and openbis IDs to map parent-child relationships
            self.openbis_id_map[object_id] = object_openbis.identifier

            # Save datasets for the object if any are attached from parsing
            self._save_datasets(object_instance, object_openbis.identifier)

        # Storing files as datasets in openBIS
        self._attach_datasets()

        # Creating relationships between objects in the collection based on the relationships defined in the collection.
        for parent, child in self.collection.relationships.values():
            parent_obj, child_obj = self._load_relationship_objs(parent, child)
            if parent_obj is None or child_obj is None:
                self.logger.warning(
                    f"Skipping relationship with parent {parent} and child {child} "
                    "because one of them is not found attached or in OpenBIS."
                )
                continue
            child_obj.add_parents(parent_obj)
            child_obj.save()
            self.logger.info(
                f"Prepared relationship: {child_obj.identifier} -> parent {parent_obj.identifier}"
            )

parsing()

Runs the parser specific class for each of the files specified in self.files_parser and adds them to the collection.

Source code in bam_masterdata/cli/run_parser.py
def parsing(self) -> None:
    """
    Runs the parser specific class for each of the files specified in `self.files_parser` and adds them to the collection.
    """
    for parser, files in self.files_parser.items():
        parser.parse(files, self.collection, logger=self.logger)

run()

Runs the complete parsing and openBIS import workflow.

Parses all configured files into the internal collection, creates or updates the corresponding objects in openBIS, stores their local-to-openBIS identifier mapping, attaches object-level and collection-level datasets, and finally creates the parent-child relationships defined during parsing.

Existing openBIS objects are updated rather than recreated when their identifier already exists.

Source code in bam_masterdata/cli/run_parser.py
def run(self) -> None:
    """
    Runs the complete parsing and openBIS import workflow.

    Parses all configured files into the internal collection, creates or updates the corresponding
    objects in openBIS, stores their local-to-openBIS identifier mapping, attaches object-level
    and collection-level datasets, and finally creates the parent-child relationships defined
    during parsing.

    Existing openBIS objects are updated rather than recreated when their identifier already exists.
    """

    # Runs the parser specific class for each of the files specified in `self.files_parser` and adds them to the `self.collection`.
    self.parsing()

    # Storing objects in openBIS and creating a mapping of local IDs to openBIS identifiers for later use in relationships.
    self.openbis_id_map = {}
    for object_id, object_instance in self.collection.attached_objects.items():
        resolved_destination = self._resolve_destination(object_id)
        identifier = self._identifier(object_instance, resolved_destination)
        obj_props = self._load_openbis_props(object_instance)

        try:
            object_openbis = self.openbis.get_object(identifier)
            self.logger.info(
                f"Object '{identifier}' already exists in openBIS, updating properties."
            )
            object_openbis.set_props(obj_props)  # update properties
        except Exception:
            self.logger.info(f"Creating new object '{object_instance.code}'.")
            object_openbis = self.openbis.new_object(
                type=object_instance.defs.code,
                code=object_instance.code,
                space=resolved_destination.space,
                project=resolved_destination.project,
                collection=(
                    resolved_destination.collection
                    if self.collection_name
                    else None
                ),
                props=obj_props,
            )
        object_openbis.save()

        # save local and openbis IDs to map parent-child relationships
        self.openbis_id_map[object_id] = object_openbis.identifier

        # Save datasets for the object if any are attached from parsing
        self._save_datasets(object_instance, object_openbis.identifier)

    # Storing files as datasets in openBIS
    self._attach_datasets()

    # Creating relationships between objects in the collection based on the relationships defined in the collection.
    for parent, child in self.collection.relationships.values():
        parent_obj, child_obj = self._load_relationship_objs(parent, child)
        if parent_obj is None or child_obj is None:
            self.logger.warning(
                f"Skipping relationship with parent {parent} and child {child} "
                "because one of them is not found attached or in OpenBIS."
            )
            continue
        child_obj.add_parents(parent_obj)
        child_obj.save()
        self.logger.info(
            f"Prepared relationship: {child_obj.identifier} -> parent {parent_obj.identifier}"
        )

RunParsersWithTransactions

Bases: RunParsers

Run parsers and persist objects and relationships to openBIS using transactions.

Object creation and updates are committed in one transaction. Relationships are committed in a separate transaction after object creation has completed.

Dataset uploads are currently performed outside transactions.

Source code in bam_masterdata/cli/run_parser.py
class RunParsersWithTransactions(RunParsers):
    """
    Run parsers and persist objects and relationships to openBIS using transactions.

    Object creation and updates are committed in one transaction. Relationships are committed in a separate
    transaction after object creation has completed.

    Dataset uploads are currently performed outside transactions.
    """

    def __init__(
        self,
        openbis: Openbis | None = None,
        space_name: str = "",
        project_name: str = "PROJECT",
        collection_name: str = "",
        files_parser: dict[AbstractParser, list[str]] | None = None,
        logger: BoundLoggerLazyProxy = logger,
        collection_type: str = "COLLECTION",
    ):
        super().__init__(
            openbis=openbis,
            space_name=space_name,
            project_name=project_name,
            collection_name=collection_name,
            files_parser=files_parser,
            logger=logger,
            collection_type=collection_type,
        )

    def run(self) -> None:
        """
        Runs the parsing and transactional openBIS import workflow.

        Parses the configured files, creates or updates all generated objects in a single transaction,
        uploads attached datasets, and finally commits parent-child relationships in a separate transaction.

        Dataset uploads are currently performed outside transactions.
        """

        self.parsing()
        self.openbis_id_map = {}

        # Create or update objects in openBIS within a single transaction
        obj_transaction = self.openbis.new_transaction()
        for object_id, object_instance in self.collection.attached_objects.items():
            resolved_destination = self._resolve_destination(object_id)
            identifier = self._identifier(object_instance, resolved_destination)
            obj_props = self._load_openbis_props(object_instance)

            try:
                object_openbis = self.openbis.get_object(identifier)
            except Exception:
                object_openbis = None

            if object_openbis is not None:
                object_openbis.set_props(obj_props)
                obj_transaction.add(object_openbis)

                self.logger.info(
                    f"Object '{identifier}' will be updated in transaction."
                )
            else:
                object_openbis = self.openbis.new_object(
                    type=object_instance.defs.code,
                    code=object_instance.code,
                    space=resolved_destination.space,
                    project=resolved_destination.project,
                    collection=(
                        resolved_destination.collection
                        if self.collection_name
                        else None
                    ),
                    props=obj_props,
                )

                obj_transaction.add(object_openbis)

                self.logger.info(
                    f"Object '{identifier}' will be created in transaction."
                )

            self.openbis_id_map[object_id] = identifier

        try:
            obj_transaction.commit()
        except Exception as exc:
            self.logger.error(f"Failed to commit object transaction: {exc}")
            return None

        self.logger.info("Object transaction committed successfully.")

        # ! Dataset transactions are not currently supported.
        # TODO (May 2026) if later transactions support datasets change it to transaction.
        for object_id, object_instance in self.collection.attached_objects.items():
            identifier = self.openbis_id_map[object_id]
            self._save_datasets(object_instance, identifier)

        self._attach_datasets()

        # Creating relationships in a separate transaction after objects have been created or updated
        rel_transaction = self.openbis.new_transaction()
        for parent, child in self.collection.relationships.values():
            parent_obj, child_obj = self._load_relationship_objs(parent, child)

            if parent_obj is None or child_obj is None:
                self.logger.warning(
                    f"Skipping relationship with parent {parent} and child {child} "
                    "because one of the objects could not be resolved."
                )
                continue

            child_obj.add_parents(parent_obj)
            rel_transaction.add(child_obj)

            self.logger.info(
                f"Prepared relationship: {child_obj.identifier} -> parent {parent_obj.identifier}"
            )
        try:
            rel_transaction.commit()
        except Exception as exc:
            self.logger.error(f"Failed to commit relationship transaction: {exc}")
            return None

        self.logger.info("Relationship transaction committed successfully.")

run()

Runs the parsing and transactional openBIS import workflow.

Parses the configured files, creates or updates all generated objects in a single transaction, uploads attached datasets, and finally commits parent-child relationships in a separate transaction.

Dataset uploads are currently performed outside transactions.

Source code in bam_masterdata/cli/run_parser.py
def run(self) -> None:
    """
    Runs the parsing and transactional openBIS import workflow.

    Parses the configured files, creates or updates all generated objects in a single transaction,
    uploads attached datasets, and finally commits parent-child relationships in a separate transaction.

    Dataset uploads are currently performed outside transactions.
    """

    self.parsing()
    self.openbis_id_map = {}

    # Create or update objects in openBIS within a single transaction
    obj_transaction = self.openbis.new_transaction()
    for object_id, object_instance in self.collection.attached_objects.items():
        resolved_destination = self._resolve_destination(object_id)
        identifier = self._identifier(object_instance, resolved_destination)
        obj_props = self._load_openbis_props(object_instance)

        try:
            object_openbis = self.openbis.get_object(identifier)
        except Exception:
            object_openbis = None

        if object_openbis is not None:
            object_openbis.set_props(obj_props)
            obj_transaction.add(object_openbis)

            self.logger.info(
                f"Object '{identifier}' will be updated in transaction."
            )
        else:
            object_openbis = self.openbis.new_object(
                type=object_instance.defs.code,
                code=object_instance.code,
                space=resolved_destination.space,
                project=resolved_destination.project,
                collection=(
                    resolved_destination.collection
                    if self.collection_name
                    else None
                ),
                props=obj_props,
            )

            obj_transaction.add(object_openbis)

            self.logger.info(
                f"Object '{identifier}' will be created in transaction."
            )

        self.openbis_id_map[object_id] = identifier

    try:
        obj_transaction.commit()
    except Exception as exc:
        self.logger.error(f"Failed to commit object transaction: {exc}")
        return None

    self.logger.info("Object transaction committed successfully.")

    # ! Dataset transactions are not currently supported.
    # TODO (May 2026) if later transactions support datasets change it to transaction.
    for object_id, object_instance in self.collection.attached_objects.items():
        identifier = self.openbis_id_map[object_id]
        self._save_datasets(object_instance, identifier)

    self._attach_datasets()

    # Creating relationships in a separate transaction after objects have been created or updated
    rel_transaction = self.openbis.new_transaction()
    for parent, child in self.collection.relationships.values():
        parent_obj, child_obj = self._load_relationship_objs(parent, child)

        if parent_obj is None or child_obj is None:
            self.logger.warning(
                f"Skipping relationship with parent {parent} and child {child} "
                "because one of the objects could not be resolved."
            )
            continue

        child_obj.add_parents(parent_obj)
        rel_transaction.add(child_obj)

        self.logger.info(
            f"Prepared relationship: {child_obj.identifier} -> parent {parent_obj.identifier}"
        )
    try:
        rel_transaction.commit()
    except Exception as exc:
        self.logger.error(f"Failed to commit relationship transaction: {exc}")
        return None

    self.logger.info("Relationship transaction committed successfully.")

bam_masterdata.excel.excel_to_entities

MasterdataExcelExtractor

Source code in bam_masterdata/excel/excel_to_entities.py
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
class MasterdataExcelExtractor:
    # TODO move these validation rules to a separate json
    VALIDATION_RULES: dict[str, dict[str, dict[str, Any]]] = {}

    def __init__(self, excel_path: str, **kwargs):
        """Initialize the MasterdataExtractor."""
        self.excel_path = excel_path
        self.row_cell_info = kwargs.get("row_cell_info", False)
        self.workbook = openpyxl.load_workbook(excel_path)
        self.logger = kwargs.get("logger", logger)

        # Load validation rules at initialization
        if not MasterdataExcelExtractor.VALIDATION_RULES:
            self.VALIDATION_RULES = load_validation_rules(
                self.logger,
                os.path.join(VALIDATION_RULES_DIR, "excel_validation_rules.json"),
            )

    def index_to_excel_column(self, index: int) -> str:
        """
        Converts a 1-based index to an Excel column name.

        Args:
            index: The 1-based index to convert.

        Returns:
            The corresponding Excel column name.
        """
        if not index >= 1:
            raise ValueError("Index must be a positive integer starting from 1.")

        column = ""
        while index > 0:
            index, remainder = divmod(index - 1, 26)
            column = chr(65 + remainder) + column
        return column

    def get_last_non_empty_row(
        self, sheet: "Worksheet", start_index: int
    ) -> int | None:
        """
        Finds the last non-empty row before encountering a completely empty row.

        Args:
            sheet: The worksheet object.
            start_index: The row number to start checking from (1-based index).

        Returns:
            The row number of the last non-empty row before an empty row is encountered,
            or None if no non-empty rows are found starting from the given index.
        """
        if start_index < 1 or start_index > sheet.max_row:
            raise ValueError(
                f"Invalid start index: {start_index}. It must be between 1 and {sheet.max_row}."
            )

        last_non_empty_row = None
        for row in range(start_index, sheet.max_row + 1):
            if all(
                sheet.cell(row=row, column=col).value in (None, "")
                for col in range(1, sheet.max_column + 1)
            ):
                return last_non_empty_row  # Return the last non-empty row before the current empty row

            last_non_empty_row = row  # Update the last non-empty row

        return last_non_empty_row  # If no empty row is encountered, return the last non-empty row

    def str_to_bool(
        self,
        value: str | bool | None,
        term: str,
        coordinate: str,
        sheet_title: str,
    ) -> bool:
        """
        Converts a string to a boolean value.

        Args:
            value: The string to convert.

        Returns:
            The boolean value.
        """
        # No `value` provided
        if not value:
            return False

        val = str(value).strip().lower()
        if val not in ["true", "false"]:
            self.logger.error(
                f"Invalid {term.lower()} value found in the {term} column at position {coordinate} in {sheet_title}. Accepted values: TRUE or FALSE.",
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
        return val == "true"

    def get_and_check_property(
        self,
        value: str | bool | None,
        term: str,
        coordinate: str,
        sheet_title: str,
        is_description: bool = False,
        is_code: bool = False,
        is_data: bool = False,
        is_url: bool = False,
    ) -> str:
        """
        Gets a property and checks its format.

        Args:
            value: The string to convert.

        Returns:
            The property.
        """

        # No `value` provided
        if not value:
            return ""

        val = str(value)
        error_message = f"Invalid {term.lower()} value found in the {term} column at position {coordinate} in {sheet_title}."
        if is_description:
            if not re.match(r".*//.*", val):
                self.logger.error(
                    error_message
                    + "Description should follow the schema: English Description + '//' + German Description. ",
                    term=term,
                    cell_value=val,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
        elif is_code:
            if not re.match(r"^\$?[A-Z0-9_.]+$", val):
                self.logger.error(
                    error_message,
                    term=term,
                    cell_value=val,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
        elif is_data:
            # Normalize data type to uppercase and allow dynamic SAMPLE/OBJECT codes
            val_upper = val.upper()
            allowed_types = [dt.value for dt in DataType]
            is_valid_standard = val_upper in allowed_types
            is_valid_dynamic = False

            if not is_valid_standard and (
                val_upper.startswith("SAMPLE:") or val_upper.startswith("OBJECT:")
            ):
                parts = val_upper.split(":", 1)
                if len(parts) == 2 and parts[1].strip():
                    is_valid_dynamic = True

            if not is_valid_standard and not is_valid_dynamic:
                self.logger.error(
                    error_message
                    + f"The Data Type should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                    term=term,
                    cell_value=val_upper,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
            val = val_upper
        elif is_url:
            if not re.match(
                r"https?://(?:www\.)?[a-zA-Z0-9-._~:/?#@!$&'()*+,;=%]+", val
            ):
                self.logger.error(
                    error_message,
                    term=term,
                    cell_value=val,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
        else:
            if not re.match(r".*", val):
                self.logger.error(
                    error_message,
                    term=term,
                    cell_value=val,
                    cell_coordinate=coordinate,
                    sheet_title=sheet_title,
                )
        return val

    # Helper function to process each term
    def process_term(
        self, term: str, cell_value: Any, coordinate: str, sheet_title: str
    ) -> Any:
        """
        Processes a term by converting it to a boolean if necessary or checking its validity.

        Args:
            term: The term being processed.
            cell_value: The value of the cell.
            coordinate: The coordinate of the cell in the sheet.
            sheet_title: The title of the sheet.

        Returns:
            The processed value, either as a boolean or the original value after validation.
        """
        # Check if the term is a boolean type
        if term in ("Mandatory", "Show in edit views"):
            return self.str_to_bool(
                value=cell_value,
                term=term,
                coordinate=coordinate,
                sheet_title=sheet_title,
            )
        # Check and validate the property
        return self.get_and_check_property(
            value=cell_value,
            term=term,
            coordinate=coordinate,
            sheet_title=sheet_title,
            is_code=(term in ["Code", "Vocabulary code"]),
            is_data=(term == "Data type"),
        )

    def extract_value(
        self,
        sheet: "Worksheet",
        row: int,
        column: int,
        validation_pattern: str = None,
        is_description: bool = False,
        is_data: bool = False,
        is_url: bool = False,
    ) -> str:
        """
        Extracts and validates a value from a specified cell in the Excel sheet.

        Args:
            sheet: The worksheet object.
            row: The row number of the cell (1-based index).
            column: The column number of the cell (1-based index).
            validation_pattern: Optional regex pattern to validate the cell value.
            is_description: Flag indicating if the value is a description.
            is_data: Flag indicating if the value is a data type.
            is_url: Flag indicating if the value is a URL.

        Returns:
            The extracted and validated cell value as a string. Returns an empty string if the value is invalid or not provided.
        """
        value = sheet.cell(row=row, column=column).value

        # No `value` provided
        if not value:
            return ""

        validated = (
            bool(re.match(validation_pattern, str(value)))
            if validation_pattern
            else True
        )
        error_message = f"Invalid value '{value}' at row {row}, column {column} in sheet {sheet.title}"

        if is_description:
            error_message += " Description should follow the schema: English Description + '//' + German Description."
        elif is_data:
            val_upper = str(value).upper()
            allowed_types = [dt.value for dt in DataType]
            is_valid_standard = val_upper in allowed_types
            is_valid_dynamic = False
            if not is_valid_standard and (
                val_upper.startswith("SAMPLE:") or val_upper.startswith("OBJECT:")
            ):
                parts = val_upper.split(":", 1)
                if len(parts) == 2 and parts[1].strip():
                    is_valid_dynamic = True
            validated = is_valid_standard or is_valid_dynamic
            error_message += f" The Data Type should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'"
        elif is_url:
            error_message += " It should be an URL or empty"

        if not validated:
            self.logger.error(
                error_message,
                cell_value=value,
                sheet_title=sheet.title,
                row=row,
                column=column,
            )

        return value or ""

    def process_entity(
        self,
        sheet: "Worksheet",
        start_index_row: int,
        header_terms: list[str],
        expected_terms: list[str],
        entity_type: str,
    ) -> dict[str, Any]:
        """
        Process an entity type block in the Excel sheet and return its attributes as a dictionary.

        Args:
            sheet: The worksheet object.
            start_index_row: The row where the current entity type begins (1-based index).
            header_terms: List of header terms in the entity block.
            expected_terms: List of expected terms to extract from the entity block.
            entity_type: The type of the entity (e.g., SAMPLE_TYPE, OBJECT_TYPE).

        Returns:
            A dictionary containing the attributes of the entity.
        """
        attributes: dict = {}
        cell_value: Any = ""

        for term in expected_terms:
            if term not in header_terms:
                self.logger.error(f"{term} not found in the headers.", term=term)
            else:
                term_index = header_terms.index(term)
                cell = sheet.cell(row=start_index_row + 2, column=term_index + 1)
                cell_value = self.extract_value(
                    sheet,
                    start_index_row + 2,
                    term_index + 1,
                    self.VALIDATION_RULES[entity_type][term].get("pattern"),
                )

                # Handle boolean conversion
                if self.VALIDATION_RULES[entity_type][term].get("is_bool"):
                    cell_value = self.str_to_bool(
                        value=cell_value,
                        term=term,
                        coordinate=cell.coordinate,
                        sheet_title=sheet.title,
                    )

                # Handle data type validation
                elif self.VALIDATION_RULES[entity_type][term].get("is_data"):
                    allowed_types = [dt.value for dt in DataType]
                    cell_value_upper = str(cell_value).upper()
                    is_valid_standard = cell_value_upper in allowed_types
                    is_valid_dynamic = False
                    if not is_valid_standard and (
                        cell_value_upper.startswith("SAMPLE:")
                        or cell_value_upper.startswith("OBJECT:")
                    ):
                        parts = cell_value_upper.split(":", 1)
                        if len(parts) == 2 and parts[1].strip():
                            is_valid_dynamic = True
                    if not is_valid_standard and not is_valid_dynamic:
                        self.logger.error(
                            f"Invalid Data Type: {cell_value} in {cell.coordinate} (Sheet: {sheet.title}). Should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                            term=term,
                            cell_value=cell_value,
                            cell_coordinate=cell.coordinate,
                            sheet_title=sheet.title,
                        )
                    else:
                        cell_value = (
                            cell_value_upper
                            if isinstance(cell_value, str)
                            else cell_value
                        )

                # Handle additional validation for "Generated code prefix"
                elif (
                    self.VALIDATION_RULES[entity_type][term].get("extra_validation")
                    == "is_reduced_version"
                ):
                    if not is_reduced_version(cell_value, attributes.get("code", "")):
                        self.logger.warning(
                            f"Invalid {term} value '{cell_value}' in {cell.coordinate} (Sheet: {sheet.title}). "
                            f"Generated code prefix should be part of the 'Code' {attributes.get('code', '')}.",
                            term=term,
                            cell_value=cell_value,
                            cell_coordinate=cell.coordinate,
                            sheet_title=sheet.title,
                        )

                # Handle validation script (allows empty but must match pattern if provided)
                elif (
                    self.VALIDATION_RULES[entity_type][term].get("allow_empty")
                    and not cell_value
                ):
                    cell_value = None

                # Handle URL template validation (allows empty but must be a valid URL)
                elif (
                    self.VALIDATION_RULES[entity_type][term].get("is_url")
                    and cell_value
                ):
                    url_pattern = self.VALIDATION_RULES[entity_type][term].get(
                        "pattern"
                    )
                    if not re.match(url_pattern, str(cell_value)):
                        self.logger.error(
                            f"Invalid URL format: {cell_value} in {cell.coordinate} (Sheet: {sheet.title})",
                            cell_value=cell_value,
                            cell_coordinate=cell.coordinate,
                            sheet_title=sheet.title,
                        )

                # Add the extracted value to the attributes dictionary
                attributes[self.VALIDATION_RULES[entity_type][term].get("key")] = (
                    cell_value
                )

        if self.row_cell_info:
            attributes["row_location"] = f"A{start_index_row}"
        return attributes

    def properties_to_dict(
        self, sheet: "Worksheet", start_index_row: int, last_non_empty_row: int
    ) -> dict[str, dict[str, Any]]:
        """
        Extracts properties from an Entity type block in the Excel sheet and returns them as a dictionary.

        Args:
            sheet: The worksheet object.
            start_index_row: Row where the current entity type begins (1-based index).
            last_non_empty_row: Row where the current entity type finish (1-based index).

        Returns:
            A dictionary where each key is a property code and the value is a dictionary
            containing the attributes of the property.
        """
        property_dict: dict = {}
        expected_terms = [
            "Code",
            "Description",
            "Mandatory",
            "Show in edit views",
            "Section",
            "Property label",
            "Data type",
            "Vocabulary code",
            "Metadata",
            "Dynamic script",
            # ! these are not used
            # "Unique",
            # "Internal assignment",
        ]

        # Determine the header row index
        header_index = start_index_row + 3
        row_headers = [(cell.value, cell.coordinate) for cell in sheet[header_index]]
        # And store how many properties are for the entity
        n_properties = last_non_empty_row - header_index
        if n_properties < 0:
            self.logger.error(
                f"No properties found for the entity in sheet {sheet.title} starting at row {start_index_row}."
            )
            return property_dict

        # Initialize a dictionary to store extracted columns
        extracted_columns: dict[str, list] = {term: [] for term in expected_terms}
        if self.row_cell_info:
            extracted_columns["row_location"] = []

        # Extract columns for each expected term
        for i, (term, coordinate) in enumerate(row_headers):
            if term not in expected_terms:
                log_func = (
                    self.logger.warning
                    if term
                    in (
                        "Mandatory",
                        "Show in edit views",
                        "Section",
                        "Metadata",
                        "Dynamic script",
                        "Vocabulary code",
                        # ! these are not used
                        # "Unique",
                        # "Internal assignment",
                    )
                    else self.logger.error
                )
                log_func(f"'{term}' not found in the properties headers.", term=term)
                continue

            # Excel column letter from the coordinate
            term_letter = coordinate[0]

            # Extract values from the column
            for cell_property in sheet[term_letter][header_index:last_non_empty_row]:
                extracted_columns[term].append(
                    self.process_term(
                        term, cell_property.value, cell_property.coordinate, sheet.title
                    )
                )
                if self.row_cell_info:
                    extracted_columns["row_location"].append(cell_property.coordinate)

        # Combine extracted values into a dictionary
        for i in range(n_properties):
            code = extracted_columns.get("Code", [])
            if not code:
                self.logger.error(
                    f"'Code' not found in the properties headers for sheet {sheet.title}."
                )
                return property_dict
            code = code[i]
            property_dict[code] = {"permId": code, "code": code}
            for key, pybis_val in {
                "Description": "description",
                "Section": "section",
                "Mandatory": "mandatory",
                "Show in edit views": "show_in_edit_views",
                "Property label": "label",
                "Data type": "dataType",
                "Vocabulary code": "vocabularyCode",
            }.items():
                data_column = extracted_columns.get(key, [])
                if not data_column:
                    continue
                cell_value = data_column[i]
                if key == "Data type":
                    object_code = None
                    normalized_value = (
                        str(cell_value).upper()
                        if isinstance(cell_value, str)
                        else cell_value
                    )
                    if isinstance(normalized_value, str) and ":" in normalized_value:
                        prefix, dynamic_code = normalized_value.split(":", 1)
                        if prefix in ("SAMPLE", "OBJECT") and dynamic_code.strip():
                            object_code = dynamic_code.strip()
                            normalized_value = DataType.OBJECT.value
                    property_dict[code][pybis_val] = normalized_value
                    if object_code:
                        property_dict[code]["objectCode"] = object_code
                else:
                    property_dict[code][pybis_val] = cell_value
            if self.row_cell_info:
                property_dict[code]["row_location"] = (
                    extracted_columns.get("row_location")[i],
                )
            # Only add optional fields if they exist in extracted_columns
            optional_fields = [
                "Metadata",
                "Dynamic script",
                "Unique",
                "Internal assignment",
            ]
            for field in optional_fields:
                if (
                    field in extracted_columns
                ):  # Check if the field exists in the extracted columns
                    if extracted_columns[field][i] == "":
                        extracted_columns[field][i] = None
                    property_dict[extracted_columns["Code"][i]][
                        field.lower().replace(" ", "_")
                    ] = extracted_columns[field][i]

        return property_dict

    def terms_to_dict(
        self, sheet: "Worksheet", start_index_row: int, last_non_empty_row: int
    ) -> dict[str, dict[str, Any]]:
        """
        Extracts terms from a Vocabulary block in the Excel sheet and returns them as a dictionary.

        Args:
            sheet: The worksheet object.
            start_index_row: Row where the current entity type begins (1-based index).
            last_non_empty_row: Row where the current entity type finish (1-based index).

        Returns:
            A dictionary where each key is a vocabulary term code and the value is a dictionary
            containing the attributes of the vocabulary term.
        """
        terms_dict = {}
        expected_terms = ["Code", "Description", "Url template", "Label", "Official"]

        header_index = start_index_row + 3
        row_headers = [cell.value for cell in sheet[header_index]]

        # Initialize a dictionary to store extracted columns
        extracted_columns: dict[str, list] = {term: [] for term in expected_terms}

        # Helper function to process each term
        def process_term_cell(term, cell_value, coordinate, sheet_title):
            if term == "Official":
                return self.str_to_bool(
                    value=cell_value,
                    term=term,
                    coordinate=coordinate,
                    sheet_title=sheet_title,
                )
            return self.get_and_check_property(
                value=cell_value,
                term=term,
                coordinate=coordinate,
                sheet_title=sheet_title,
                is_code=(term == "Code"),
                is_url=(term == "Url template"),
            )

        # Extract columns for each expected term
        for term in expected_terms:
            if term not in row_headers:
                self.logger.warning(
                    f"{term} not found in the properties headers.", term=term
                )
                continue

            # Get column index and Excel letter
            term_index = row_headers.index(term) + 1
            term_letter = self.index_to_excel_column(term_index)

            # Extract values from the column
            for cell in sheet[term_letter][header_index:last_non_empty_row]:
                extracted_columns[term].append(
                    process_term_cell(term, cell.value, cell.coordinate, sheet.title)
                )

        if not extracted_columns.get("Code"):
            self.logger.error(
                f"The required 'Code' column for terms was not found in sheet {sheet.title}."
            )
            return {}

        # Combine extracted values into a dictionary safely
        for i in range(len(extracted_columns["Code"])):
            code = extracted_columns["Code"][i]
            terms_dict[code] = {
                "permId": code,
                "code": code,
            }
            for key, pybis_val in {
                "Description": "descriptions",
                "Url template": "url_template",
                "Label": "label",
                "Official": "official",
            }.items():
                values = extracted_columns.get(key, [])
                if len(values) > i:
                    terms_dict[code][pybis_val] = values[i]

        return terms_dict

    def block_to_entity_dict(
        self,
        sheet: "Worksheet",
        start_index_row: int,
        last_non_empty_row: int,
        complete_dict: dict[str, Any],
    ) -> dict[str, Any]:
        """
        Extracts entity attributes from an Excel sheet block and returns them as a dictionary.
        """
        attributes_dict: dict = {}

        # Get the entity type
        entity_type = sheet[f"A{start_index_row}"].value
        if entity_type not in self.VALIDATION_RULES:
            raise ValueError(f"Invalid entity type: {entity_type}")

        # Get the header terms
        header_terms = [cell.value for cell in sheet[start_index_row + 1]]

        # Process entity data using the helper function
        attributes_dict = self.process_entity(
            sheet,
            start_index_row,
            header_terms,
            list(self.VALIDATION_RULES[entity_type].keys()),
            entity_type,
        )

        # Extract additional attributes if necessary
        if entity_type in {
            "SAMPLE_TYPE",
            "OBJECT_TYPE",
            "EXPERIMENT_TYPE",
            "DATASET_TYPE",
        }:
            attributes_dict["properties"] = (
                self.properties_to_dict(sheet, start_index_row, last_non_empty_row)
                or {}
            )

        elif entity_type == "VOCABULARY_TYPE":
            attributes_dict["terms"] = (
                self.terms_to_dict(sheet, start_index_row, last_non_empty_row) or {}
            )

        # Add the entity to the complete dictionary
        complete_dict[attributes_dict["code"]] = attributes_dict

        # Return sorted dictionary
        return dict(sorted(complete_dict.items(), key=lambda item: item[0].count(".")))

    def excel_to_entities(self) -> dict[str, dict[str, Any]]:
        """
        Extracts entities from an Excel file and returns them as a dictionary.

        Returns:
            dict[str, dict[str, Any]]: A dictionary where each key is a normalized sheet name and the value is a dictionary
            containing the extracted entities. Returns an empty dictionary if all sheets are empty.
        """
        sheets_dict: dict[str, dict[str, Any]] = {}
        sheet_names = self.workbook.sheetnames
        has_content = False  # Track if any sheet has valid content

        for i, sheet_name in enumerate(sheet_names):
            normalized_sheet_name = sheet_name.lower().replace(" ", "_")
            sheet = self.workbook[sheet_name]
            start_row = 1

            # **Check if the sheet is empty**
            if all(
                sheet.cell(row=row, column=col).value in (None, "")
                for row in range(1, sheet.max_row + 1)
                for col in range(1, sheet.max_column + 1)
            ):
                self.logger.info(f"Skipping empty sheet: {sheet_name}")
                continue  # Move to the next sheet

            sheets_dict[normalized_sheet_name] = {}

            consecutive_empty_rows = 0  # Track consecutive empty rows
            while start_row <= sheet.max_row:
                # **Check for two consecutive empty rows**
                is_row_empty = all(
                    sheet.cell(row=start_row, column=col).value in (None, "")
                    for col in range(1, sheet.max_column + 1)
                )

                if is_row_empty:
                    consecutive_empty_rows += 1
                    if consecutive_empty_rows >= 2:
                        # **Reached the end of the sheet, move to the next**
                        if i == len(sheet_names) - 1:
                            self.logger.info(
                                f"Last sheet {sheet_name} processed. End of the file reached."
                            )
                        else:
                            self.logger.info(
                                f"End of the current sheet {sheet_name} reached. Switching to next sheet..."
                            )
                        break  # Stop processing this sheet
                else:
                    consecutive_empty_rows = 0  # Reset if we find a non-empty row

                    # **Process the entity block**
                    last_non_empty_row = self.get_last_non_empty_row(sheet, start_row)
                    if last_non_empty_row is None:
                        break  # No more valid blocks

                    sheets_dict[normalized_sheet_name] = self.block_to_entity_dict(
                        sheet,
                        start_row,
                        last_non_empty_row,
                        sheets_dict[normalized_sheet_name],
                    )
                    has_content = True  # Found valid content

                    # Move to the next entity block
                    start_row = last_non_empty_row + 1
                    continue  # Continue loop without increasing consecutive_empty_rows

                start_row += 1  # Move to the next row

        # **If no sheets had content, return an empty dictionary**
        if not has_content:
            self.logger.warning(
                "No valid data found in any sheets. Returning empty dictionary."
            )
            return {}

        return sheets_dict

__init__(excel_path, **kwargs)

Initialize the MasterdataExtractor.

Source code in bam_masterdata/excel/excel_to_entities.py
def __init__(self, excel_path: str, **kwargs):
    """Initialize the MasterdataExtractor."""
    self.excel_path = excel_path
    self.row_cell_info = kwargs.get("row_cell_info", False)
    self.workbook = openpyxl.load_workbook(excel_path)
    self.logger = kwargs.get("logger", logger)

    # Load validation rules at initialization
    if not MasterdataExcelExtractor.VALIDATION_RULES:
        self.VALIDATION_RULES = load_validation_rules(
            self.logger,
            os.path.join(VALIDATION_RULES_DIR, "excel_validation_rules.json"),
        )

block_to_entity_dict(sheet, start_index_row, last_non_empty_row, complete_dict)

Extracts entity attributes from an Excel sheet block and returns them as a dictionary.

Source code in bam_masterdata/excel/excel_to_entities.py
def block_to_entity_dict(
    self,
    sheet: "Worksheet",
    start_index_row: int,
    last_non_empty_row: int,
    complete_dict: dict[str, Any],
) -> dict[str, Any]:
    """
    Extracts entity attributes from an Excel sheet block and returns them as a dictionary.
    """
    attributes_dict: dict = {}

    # Get the entity type
    entity_type = sheet[f"A{start_index_row}"].value
    if entity_type not in self.VALIDATION_RULES:
        raise ValueError(f"Invalid entity type: {entity_type}")

    # Get the header terms
    header_terms = [cell.value for cell in sheet[start_index_row + 1]]

    # Process entity data using the helper function
    attributes_dict = self.process_entity(
        sheet,
        start_index_row,
        header_terms,
        list(self.VALIDATION_RULES[entity_type].keys()),
        entity_type,
    )

    # Extract additional attributes if necessary
    if entity_type in {
        "SAMPLE_TYPE",
        "OBJECT_TYPE",
        "EXPERIMENT_TYPE",
        "DATASET_TYPE",
    }:
        attributes_dict["properties"] = (
            self.properties_to_dict(sheet, start_index_row, last_non_empty_row)
            or {}
        )

    elif entity_type == "VOCABULARY_TYPE":
        attributes_dict["terms"] = (
            self.terms_to_dict(sheet, start_index_row, last_non_empty_row) or {}
        )

    # Add the entity to the complete dictionary
    complete_dict[attributes_dict["code"]] = attributes_dict

    # Return sorted dictionary
    return dict(sorted(complete_dict.items(), key=lambda item: item[0].count(".")))

excel_to_entities()

Extracts entities from an Excel file and returns them as a dictionary.

Returns:

Type Description
dict[str, dict[str, Any]]

dict[str, dict[str, Any]]: A dictionary where each key is a normalized sheet name and the value is a dictionary

dict[str, dict[str, Any]]

containing the extracted entities. Returns an empty dictionary if all sheets are empty.

Source code in bam_masterdata/excel/excel_to_entities.py
def excel_to_entities(self) -> dict[str, dict[str, Any]]:
    """
    Extracts entities from an Excel file and returns them as a dictionary.

    Returns:
        dict[str, dict[str, Any]]: A dictionary where each key is a normalized sheet name and the value is a dictionary
        containing the extracted entities. Returns an empty dictionary if all sheets are empty.
    """
    sheets_dict: dict[str, dict[str, Any]] = {}
    sheet_names = self.workbook.sheetnames
    has_content = False  # Track if any sheet has valid content

    for i, sheet_name in enumerate(sheet_names):
        normalized_sheet_name = sheet_name.lower().replace(" ", "_")
        sheet = self.workbook[sheet_name]
        start_row = 1

        # **Check if the sheet is empty**
        if all(
            sheet.cell(row=row, column=col).value in (None, "")
            for row in range(1, sheet.max_row + 1)
            for col in range(1, sheet.max_column + 1)
        ):
            self.logger.info(f"Skipping empty sheet: {sheet_name}")
            continue  # Move to the next sheet

        sheets_dict[normalized_sheet_name] = {}

        consecutive_empty_rows = 0  # Track consecutive empty rows
        while start_row <= sheet.max_row:
            # **Check for two consecutive empty rows**
            is_row_empty = all(
                sheet.cell(row=start_row, column=col).value in (None, "")
                for col in range(1, sheet.max_column + 1)
            )

            if is_row_empty:
                consecutive_empty_rows += 1
                if consecutive_empty_rows >= 2:
                    # **Reached the end of the sheet, move to the next**
                    if i == len(sheet_names) - 1:
                        self.logger.info(
                            f"Last sheet {sheet_name} processed. End of the file reached."
                        )
                    else:
                        self.logger.info(
                            f"End of the current sheet {sheet_name} reached. Switching to next sheet..."
                        )
                    break  # Stop processing this sheet
            else:
                consecutive_empty_rows = 0  # Reset if we find a non-empty row

                # **Process the entity block**
                last_non_empty_row = self.get_last_non_empty_row(sheet, start_row)
                if last_non_empty_row is None:
                    break  # No more valid blocks

                sheets_dict[normalized_sheet_name] = self.block_to_entity_dict(
                    sheet,
                    start_row,
                    last_non_empty_row,
                    sheets_dict[normalized_sheet_name],
                )
                has_content = True  # Found valid content

                # Move to the next entity block
                start_row = last_non_empty_row + 1
                continue  # Continue loop without increasing consecutive_empty_rows

            start_row += 1  # Move to the next row

    # **If no sheets had content, return an empty dictionary**
    if not has_content:
        self.logger.warning(
            "No valid data found in any sheets. Returning empty dictionary."
        )
        return {}

    return sheets_dict

extract_value(sheet, row, column, validation_pattern=None, is_description=False, is_data=False, is_url=False)

Extracts and validates a value from a specified cell in the Excel sheet.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
row int

The row number of the cell (1-based index).

required
column int

The column number of the cell (1-based index).

required
validation_pattern str

Optional regex pattern to validate the cell value.

None
is_description bool

Flag indicating if the value is a description.

False
is_data bool

Flag indicating if the value is a data type.

False
is_url bool

Flag indicating if the value is a URL.

False

Returns:

Type Description
str

The extracted and validated cell value as a string. Returns an empty string if the value is invalid or not provided.

Source code in bam_masterdata/excel/excel_to_entities.py
def extract_value(
    self,
    sheet: "Worksheet",
    row: int,
    column: int,
    validation_pattern: str = None,
    is_description: bool = False,
    is_data: bool = False,
    is_url: bool = False,
) -> str:
    """
    Extracts and validates a value from a specified cell in the Excel sheet.

    Args:
        sheet: The worksheet object.
        row: The row number of the cell (1-based index).
        column: The column number of the cell (1-based index).
        validation_pattern: Optional regex pattern to validate the cell value.
        is_description: Flag indicating if the value is a description.
        is_data: Flag indicating if the value is a data type.
        is_url: Flag indicating if the value is a URL.

    Returns:
        The extracted and validated cell value as a string. Returns an empty string if the value is invalid or not provided.
    """
    value = sheet.cell(row=row, column=column).value

    # No `value` provided
    if not value:
        return ""

    validated = (
        bool(re.match(validation_pattern, str(value)))
        if validation_pattern
        else True
    )
    error_message = f"Invalid value '{value}' at row {row}, column {column} in sheet {sheet.title}"

    if is_description:
        error_message += " Description should follow the schema: English Description + '//' + German Description."
    elif is_data:
        val_upper = str(value).upper()
        allowed_types = [dt.value for dt in DataType]
        is_valid_standard = val_upper in allowed_types
        is_valid_dynamic = False
        if not is_valid_standard and (
            val_upper.startswith("SAMPLE:") or val_upper.startswith("OBJECT:")
        ):
            parts = val_upper.split(":", 1)
            if len(parts) == 2 and parts[1].strip():
                is_valid_dynamic = True
        validated = is_valid_standard or is_valid_dynamic
        error_message += f" The Data Type should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'"
    elif is_url:
        error_message += " It should be an URL or empty"

    if not validated:
        self.logger.error(
            error_message,
            cell_value=value,
            sheet_title=sheet.title,
            row=row,
            column=column,
        )

    return value or ""

get_and_check_property(value, term, coordinate, sheet_title, is_description=False, is_code=False, is_data=False, is_url=False)

Gets a property and checks its format.

Parameters:

Name Type Description Default
value str | bool | None

The string to convert.

required

Returns:

Type Description
str

The property.

Source code in bam_masterdata/excel/excel_to_entities.py
def get_and_check_property(
    self,
    value: str | bool | None,
    term: str,
    coordinate: str,
    sheet_title: str,
    is_description: bool = False,
    is_code: bool = False,
    is_data: bool = False,
    is_url: bool = False,
) -> str:
    """
    Gets a property and checks its format.

    Args:
        value: The string to convert.

    Returns:
        The property.
    """

    # No `value` provided
    if not value:
        return ""

    val = str(value)
    error_message = f"Invalid {term.lower()} value found in the {term} column at position {coordinate} in {sheet_title}."
    if is_description:
        if not re.match(r".*//.*", val):
            self.logger.error(
                error_message
                + "Description should follow the schema: English Description + '//' + German Description. ",
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
    elif is_code:
        if not re.match(r"^\$?[A-Z0-9_.]+$", val):
            self.logger.error(
                error_message,
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
    elif is_data:
        # Normalize data type to uppercase and allow dynamic SAMPLE/OBJECT codes
        val_upper = val.upper()
        allowed_types = [dt.value for dt in DataType]
        is_valid_standard = val_upper in allowed_types
        is_valid_dynamic = False

        if not is_valid_standard and (
            val_upper.startswith("SAMPLE:") or val_upper.startswith("OBJECT:")
        ):
            parts = val_upper.split(":", 1)
            if len(parts) == 2 and parts[1].strip():
                is_valid_dynamic = True

        if not is_valid_standard and not is_valid_dynamic:
            self.logger.error(
                error_message
                + f"The Data Type should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                term=term,
                cell_value=val_upper,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
        val = val_upper
    elif is_url:
        if not re.match(
            r"https?://(?:www\.)?[a-zA-Z0-9-._~:/?#@!$&'()*+,;=%]+", val
        ):
            self.logger.error(
                error_message,
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
    else:
        if not re.match(r".*", val):
            self.logger.error(
                error_message,
                term=term,
                cell_value=val,
                cell_coordinate=coordinate,
                sheet_title=sheet_title,
            )
    return val

get_last_non_empty_row(sheet, start_index)

Finds the last non-empty row before encountering a completely empty row.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
start_index int

The row number to start checking from (1-based index).

required

Returns:

Type Description
int | None

The row number of the last non-empty row before an empty row is encountered,

int | None

or None if no non-empty rows are found starting from the given index.

Source code in bam_masterdata/excel/excel_to_entities.py
def get_last_non_empty_row(
    self, sheet: "Worksheet", start_index: int
) -> int | None:
    """
    Finds the last non-empty row before encountering a completely empty row.

    Args:
        sheet: The worksheet object.
        start_index: The row number to start checking from (1-based index).

    Returns:
        The row number of the last non-empty row before an empty row is encountered,
        or None if no non-empty rows are found starting from the given index.
    """
    if start_index < 1 or start_index > sheet.max_row:
        raise ValueError(
            f"Invalid start index: {start_index}. It must be between 1 and {sheet.max_row}."
        )

    last_non_empty_row = None
    for row in range(start_index, sheet.max_row + 1):
        if all(
            sheet.cell(row=row, column=col).value in (None, "")
            for col in range(1, sheet.max_column + 1)
        ):
            return last_non_empty_row  # Return the last non-empty row before the current empty row

        last_non_empty_row = row  # Update the last non-empty row

    return last_non_empty_row  # If no empty row is encountered, return the last non-empty row

index_to_excel_column(index)

Converts a 1-based index to an Excel column name.

Parameters:

Name Type Description Default
index int

The 1-based index to convert.

required

Returns:

Type Description
str

The corresponding Excel column name.

Source code in bam_masterdata/excel/excel_to_entities.py
def index_to_excel_column(self, index: int) -> str:
    """
    Converts a 1-based index to an Excel column name.

    Args:
        index: The 1-based index to convert.

    Returns:
        The corresponding Excel column name.
    """
    if not index >= 1:
        raise ValueError("Index must be a positive integer starting from 1.")

    column = ""
    while index > 0:
        index, remainder = divmod(index - 1, 26)
        column = chr(65 + remainder) + column
    return column

process_entity(sheet, start_index_row, header_terms, expected_terms, entity_type)

Process an entity type block in the Excel sheet and return its attributes as a dictionary.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
start_index_row int

The row where the current entity type begins (1-based index).

required
header_terms list[str]

List of header terms in the entity block.

required
expected_terms list[str]

List of expected terms to extract from the entity block.

required
entity_type str

The type of the entity (e.g., SAMPLE_TYPE, OBJECT_TYPE).

required

Returns:

Type Description
dict[str, Any]

A dictionary containing the attributes of the entity.

Source code in bam_masterdata/excel/excel_to_entities.py
def process_entity(
    self,
    sheet: "Worksheet",
    start_index_row: int,
    header_terms: list[str],
    expected_terms: list[str],
    entity_type: str,
) -> dict[str, Any]:
    """
    Process an entity type block in the Excel sheet and return its attributes as a dictionary.

    Args:
        sheet: The worksheet object.
        start_index_row: The row where the current entity type begins (1-based index).
        header_terms: List of header terms in the entity block.
        expected_terms: List of expected terms to extract from the entity block.
        entity_type: The type of the entity (e.g., SAMPLE_TYPE, OBJECT_TYPE).

    Returns:
        A dictionary containing the attributes of the entity.
    """
    attributes: dict = {}
    cell_value: Any = ""

    for term in expected_terms:
        if term not in header_terms:
            self.logger.error(f"{term} not found in the headers.", term=term)
        else:
            term_index = header_terms.index(term)
            cell = sheet.cell(row=start_index_row + 2, column=term_index + 1)
            cell_value = self.extract_value(
                sheet,
                start_index_row + 2,
                term_index + 1,
                self.VALIDATION_RULES[entity_type][term].get("pattern"),
            )

            # Handle boolean conversion
            if self.VALIDATION_RULES[entity_type][term].get("is_bool"):
                cell_value = self.str_to_bool(
                    value=cell_value,
                    term=term,
                    coordinate=cell.coordinate,
                    sheet_title=sheet.title,
                )

            # Handle data type validation
            elif self.VALIDATION_RULES[entity_type][term].get("is_data"):
                allowed_types = [dt.value for dt in DataType]
                cell_value_upper = str(cell_value).upper()
                is_valid_standard = cell_value_upper in allowed_types
                is_valid_dynamic = False
                if not is_valid_standard and (
                    cell_value_upper.startswith("SAMPLE:")
                    or cell_value_upper.startswith("OBJECT:")
                ):
                    parts = cell_value_upper.split(":", 1)
                    if len(parts) == 2 and parts[1].strip():
                        is_valid_dynamic = True
                if not is_valid_standard and not is_valid_dynamic:
                    self.logger.error(
                        f"Invalid Data Type: {cell_value} in {cell.coordinate} (Sheet: {sheet.title}). Should be one of the following: {allowed_types} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                        term=term,
                        cell_value=cell_value,
                        cell_coordinate=cell.coordinate,
                        sheet_title=sheet.title,
                    )
                else:
                    cell_value = (
                        cell_value_upper
                        if isinstance(cell_value, str)
                        else cell_value
                    )

            # Handle additional validation for "Generated code prefix"
            elif (
                self.VALIDATION_RULES[entity_type][term].get("extra_validation")
                == "is_reduced_version"
            ):
                if not is_reduced_version(cell_value, attributes.get("code", "")):
                    self.logger.warning(
                        f"Invalid {term} value '{cell_value}' in {cell.coordinate} (Sheet: {sheet.title}). "
                        f"Generated code prefix should be part of the 'Code' {attributes.get('code', '')}.",
                        term=term,
                        cell_value=cell_value,
                        cell_coordinate=cell.coordinate,
                        sheet_title=sheet.title,
                    )

            # Handle validation script (allows empty but must match pattern if provided)
            elif (
                self.VALIDATION_RULES[entity_type][term].get("allow_empty")
                and not cell_value
            ):
                cell_value = None

            # Handle URL template validation (allows empty but must be a valid URL)
            elif (
                self.VALIDATION_RULES[entity_type][term].get("is_url")
                and cell_value
            ):
                url_pattern = self.VALIDATION_RULES[entity_type][term].get(
                    "pattern"
                )
                if not re.match(url_pattern, str(cell_value)):
                    self.logger.error(
                        f"Invalid URL format: {cell_value} in {cell.coordinate} (Sheet: {sheet.title})",
                        cell_value=cell_value,
                        cell_coordinate=cell.coordinate,
                        sheet_title=sheet.title,
                    )

            # Add the extracted value to the attributes dictionary
            attributes[self.VALIDATION_RULES[entity_type][term].get("key")] = (
                cell_value
            )

    if self.row_cell_info:
        attributes["row_location"] = f"A{start_index_row}"
    return attributes

process_term(term, cell_value, coordinate, sheet_title)

Processes a term by converting it to a boolean if necessary or checking its validity.

Parameters:

Name Type Description Default
term str

The term being processed.

required
cell_value Any

The value of the cell.

required
coordinate str

The coordinate of the cell in the sheet.

required
sheet_title str

The title of the sheet.

required

Returns:

Type Description
Any

The processed value, either as a boolean or the original value after validation.

Source code in bam_masterdata/excel/excel_to_entities.py
def process_term(
    self, term: str, cell_value: Any, coordinate: str, sheet_title: str
) -> Any:
    """
    Processes a term by converting it to a boolean if necessary or checking its validity.

    Args:
        term: The term being processed.
        cell_value: The value of the cell.
        coordinate: The coordinate of the cell in the sheet.
        sheet_title: The title of the sheet.

    Returns:
        The processed value, either as a boolean or the original value after validation.
    """
    # Check if the term is a boolean type
    if term in ("Mandatory", "Show in edit views"):
        return self.str_to_bool(
            value=cell_value,
            term=term,
            coordinate=coordinate,
            sheet_title=sheet_title,
        )
    # Check and validate the property
    return self.get_and_check_property(
        value=cell_value,
        term=term,
        coordinate=coordinate,
        sheet_title=sheet_title,
        is_code=(term in ["Code", "Vocabulary code"]),
        is_data=(term == "Data type"),
    )

properties_to_dict(sheet, start_index_row, last_non_empty_row)

Extracts properties from an Entity type block in the Excel sheet and returns them as a dictionary.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
start_index_row int

Row where the current entity type begins (1-based index).

required
last_non_empty_row int

Row where the current entity type finish (1-based index).

required

Returns:

Type Description
dict[str, dict[str, Any]]

A dictionary where each key is a property code and the value is a dictionary

dict[str, dict[str, Any]]

containing the attributes of the property.

Source code in bam_masterdata/excel/excel_to_entities.py
def properties_to_dict(
    self, sheet: "Worksheet", start_index_row: int, last_non_empty_row: int
) -> dict[str, dict[str, Any]]:
    """
    Extracts properties from an Entity type block in the Excel sheet and returns them as a dictionary.

    Args:
        sheet: The worksheet object.
        start_index_row: Row where the current entity type begins (1-based index).
        last_non_empty_row: Row where the current entity type finish (1-based index).

    Returns:
        A dictionary where each key is a property code and the value is a dictionary
        containing the attributes of the property.
    """
    property_dict: dict = {}
    expected_terms = [
        "Code",
        "Description",
        "Mandatory",
        "Show in edit views",
        "Section",
        "Property label",
        "Data type",
        "Vocabulary code",
        "Metadata",
        "Dynamic script",
        # ! these are not used
        # "Unique",
        # "Internal assignment",
    ]

    # Determine the header row index
    header_index = start_index_row + 3
    row_headers = [(cell.value, cell.coordinate) for cell in sheet[header_index]]
    # And store how many properties are for the entity
    n_properties = last_non_empty_row - header_index
    if n_properties < 0:
        self.logger.error(
            f"No properties found for the entity in sheet {sheet.title} starting at row {start_index_row}."
        )
        return property_dict

    # Initialize a dictionary to store extracted columns
    extracted_columns: dict[str, list] = {term: [] for term in expected_terms}
    if self.row_cell_info:
        extracted_columns["row_location"] = []

    # Extract columns for each expected term
    for i, (term, coordinate) in enumerate(row_headers):
        if term not in expected_terms:
            log_func = (
                self.logger.warning
                if term
                in (
                    "Mandatory",
                    "Show in edit views",
                    "Section",
                    "Metadata",
                    "Dynamic script",
                    "Vocabulary code",
                    # ! these are not used
                    # "Unique",
                    # "Internal assignment",
                )
                else self.logger.error
            )
            log_func(f"'{term}' not found in the properties headers.", term=term)
            continue

        # Excel column letter from the coordinate
        term_letter = coordinate[0]

        # Extract values from the column
        for cell_property in sheet[term_letter][header_index:last_non_empty_row]:
            extracted_columns[term].append(
                self.process_term(
                    term, cell_property.value, cell_property.coordinate, sheet.title
                )
            )
            if self.row_cell_info:
                extracted_columns["row_location"].append(cell_property.coordinate)

    # Combine extracted values into a dictionary
    for i in range(n_properties):
        code = extracted_columns.get("Code", [])
        if not code:
            self.logger.error(
                f"'Code' not found in the properties headers for sheet {sheet.title}."
            )
            return property_dict
        code = code[i]
        property_dict[code] = {"permId": code, "code": code}
        for key, pybis_val in {
            "Description": "description",
            "Section": "section",
            "Mandatory": "mandatory",
            "Show in edit views": "show_in_edit_views",
            "Property label": "label",
            "Data type": "dataType",
            "Vocabulary code": "vocabularyCode",
        }.items():
            data_column = extracted_columns.get(key, [])
            if not data_column:
                continue
            cell_value = data_column[i]
            if key == "Data type":
                object_code = None
                normalized_value = (
                    str(cell_value).upper()
                    if isinstance(cell_value, str)
                    else cell_value
                )
                if isinstance(normalized_value, str) and ":" in normalized_value:
                    prefix, dynamic_code = normalized_value.split(":", 1)
                    if prefix in ("SAMPLE", "OBJECT") and dynamic_code.strip():
                        object_code = dynamic_code.strip()
                        normalized_value = DataType.OBJECT.value
                property_dict[code][pybis_val] = normalized_value
                if object_code:
                    property_dict[code]["objectCode"] = object_code
            else:
                property_dict[code][pybis_val] = cell_value
        if self.row_cell_info:
            property_dict[code]["row_location"] = (
                extracted_columns.get("row_location")[i],
            )
        # Only add optional fields if they exist in extracted_columns
        optional_fields = [
            "Metadata",
            "Dynamic script",
            "Unique",
            "Internal assignment",
        ]
        for field in optional_fields:
            if (
                field in extracted_columns
            ):  # Check if the field exists in the extracted columns
                if extracted_columns[field][i] == "":
                    extracted_columns[field][i] = None
                property_dict[extracted_columns["Code"][i]][
                    field.lower().replace(" ", "_")
                ] = extracted_columns[field][i]

    return property_dict

str_to_bool(value, term, coordinate, sheet_title)

Converts a string to a boolean value.

Parameters:

Name Type Description Default
value str | bool | None

The string to convert.

required

Returns:

Type Description
bool

The boolean value.

Source code in bam_masterdata/excel/excel_to_entities.py
def str_to_bool(
    self,
    value: str | bool | None,
    term: str,
    coordinate: str,
    sheet_title: str,
) -> bool:
    """
    Converts a string to a boolean value.

    Args:
        value: The string to convert.

    Returns:
        The boolean value.
    """
    # No `value` provided
    if not value:
        return False

    val = str(value).strip().lower()
    if val not in ["true", "false"]:
        self.logger.error(
            f"Invalid {term.lower()} value found in the {term} column at position {coordinate} in {sheet_title}. Accepted values: TRUE or FALSE.",
            term=term,
            cell_value=val,
            cell_coordinate=coordinate,
            sheet_title=sheet_title,
        )
    return val == "true"

terms_to_dict(sheet, start_index_row, last_non_empty_row)

Extracts terms from a Vocabulary block in the Excel sheet and returns them as a dictionary.

Parameters:

Name Type Description Default
sheet Worksheet

The worksheet object.

required
start_index_row int

Row where the current entity type begins (1-based index).

required
last_non_empty_row int

Row where the current entity type finish (1-based index).

required

Returns:

Type Description
dict[str, dict[str, Any]]

A dictionary where each key is a vocabulary term code and the value is a dictionary

dict[str, dict[str, Any]]

containing the attributes of the vocabulary term.

Source code in bam_masterdata/excel/excel_to_entities.py
def terms_to_dict(
    self, sheet: "Worksheet", start_index_row: int, last_non_empty_row: int
) -> dict[str, dict[str, Any]]:
    """
    Extracts terms from a Vocabulary block in the Excel sheet and returns them as a dictionary.

    Args:
        sheet: The worksheet object.
        start_index_row: Row where the current entity type begins (1-based index).
        last_non_empty_row: Row where the current entity type finish (1-based index).

    Returns:
        A dictionary where each key is a vocabulary term code and the value is a dictionary
        containing the attributes of the vocabulary term.
    """
    terms_dict = {}
    expected_terms = ["Code", "Description", "Url template", "Label", "Official"]

    header_index = start_index_row + 3
    row_headers = [cell.value for cell in sheet[header_index]]

    # Initialize a dictionary to store extracted columns
    extracted_columns: dict[str, list] = {term: [] for term in expected_terms}

    # Helper function to process each term
    def process_term_cell(term, cell_value, coordinate, sheet_title):
        if term == "Official":
            return self.str_to_bool(
                value=cell_value,
                term=term,
                coordinate=coordinate,
                sheet_title=sheet_title,
            )
        return self.get_and_check_property(
            value=cell_value,
            term=term,
            coordinate=coordinate,
            sheet_title=sheet_title,
            is_code=(term == "Code"),
            is_url=(term == "Url template"),
        )

    # Extract columns for each expected term
    for term in expected_terms:
        if term not in row_headers:
            self.logger.warning(
                f"{term} not found in the properties headers.", term=term
            )
            continue

        # Get column index and Excel letter
        term_index = row_headers.index(term) + 1
        term_letter = self.index_to_excel_column(term_index)

        # Extract values from the column
        for cell in sheet[term_letter][header_index:last_non_empty_row]:
            extracted_columns[term].append(
                process_term_cell(term, cell.value, cell.coordinate, sheet.title)
            )

    if not extracted_columns.get("Code"):
        self.logger.error(
            f"The required 'Code' column for terms was not found in sheet {sheet.title}."
        )
        return {}

    # Combine extracted values into a dictionary safely
    for i in range(len(extracted_columns["Code"])):
        code = extracted_columns["Code"][i]
        terms_dict[code] = {
            "permId": code,
            "code": code,
        }
        for key, pybis_val in {
            "Description": "descriptions",
            "Url template": "url_template",
            "Label": "label",
            "Official": "official",
        }.items():
            values = extracted_columns.get(key, [])
            if len(values) > i:
                terms_dict[code][pybis_val] = values[i]

    return terms_dict

bam_masterdata.openbis.login

ologin(url='')

Connect to openBIS using the credentials stored in the environment variables.

If an existing Openbis session is provided, the session is returned.

Parameters:

Name Type Description Default
url (str, Openbis)

The URL of the openBIS instance. Defaults to the value of the OPENBIS_URL environment variable.

''

Returns:

Name Type Description
Openbis Openbis

Openbis object for the specific openBIS instance defined in URL.

Source code in bam_masterdata/openbis/login.py
def ologin(url: str | Openbis = "") -> Openbis:
    """
    Connect to openBIS using the credentials stored in the environment variables.

    If an existing Openbis session is provided, the session is returned.

    Args:
        url (str, Openbis): The URL of the openBIS instance. Defaults to the value of the `OPENBIS_URL` environment variable.
        An existing Openbis session can be provided.

    Returns:
        Openbis: Openbis object for the specific openBIS instance defined in `URL`.
    """
    if isinstance(url, Openbis):
        return url

    o = Openbis(url)
    o.login(
        environ("OPENBIS_USERNAME"),
        environ("OPENBIS_PASSWORD"),
        save_token=True,
    )

    return o

bam_masterdata.openbis.get_entities

OpenbisEntities

Class to get openBIS entities and their attributes as dictionaries to be printed in the Python modules of bam_masterdata/datamodel/.

Source code in bam_masterdata/openbis/get_entities.py
class OpenbisEntities:
    """
    Class to get openBIS entities and their attributes as dictionaries to be printed in the
    Python modules of `bam_masterdata/datamodel/`.
    """

    def __init__(self, url: str | Openbis = ""):
        self.openbis = ologin(url=url)

    def _get_formatted_dict(self, entity_name: str):
        # entity_name is property_types, collection_types, dataset_types, object_types, or vocabularies
        entity_types = getattr(self.openbis, f"get_{entity_name}")().df.to_dict(
            orient="records"
        )
        return {entry["code"]: entry for entry in entity_types}

    def _assign_properties(self, entity_name: str, formatted_dict: dict) -> None:
        for entity in getattr(self.openbis, f"get_{entity_name}")():
            perm_id = entity.permId  # Unique identifier for the entity
            assignments = entity.get_property_assignments()

            if assignments:
                # Convert property assignments to list of dictionaries
                assignments_dict = assignments.df.to_dict(orient="records")

                # Create a dictionary of properties using the correct permId
                properties = {}
                for entry in assignments_dict:
                    property_perm_id = self.openbis.get_property_type(
                        entry.get("code", {})
                    ).permId
                    if property_perm_id:
                        # Include the desired property fields
                        properties[property_perm_id] = {
                            "@type": entry.get(
                                "@type", "as.dto.property.PropertyAssignment"
                            ),
                            "@id": entry.get("@id", None),
                            "fetchOptions": entry.get("fetchOptions", None),
                            "permId": property_perm_id,
                            "section": entry.get("section", ""),
                            "ordinal": entry.get("ordinal", None),
                            "mandatory": entry.get("mandatory", False),
                            "showInEditView": entry.get("showInEditView", False),
                            "showRawValueInForms": entry.get(
                                "showRawValueInForms", False
                            ),
                            "semanticAnnotations": entry.get(
                                "semanticAnnotations", None
                            ),
                            "semanticAnnotationsInherited": entry.get(
                                "semanticAnnotationsInherited", False
                            ),
                            "registrator": entry.get("registrator", None),
                            "registrationDate": entry.get("registrationDate", None),
                            "plugin": entry.get("plugin", ""),
                        }
                for prop in assignments:
                    prop = prop.get_property_type()
                    properties[prop.permId].update(
                        {
                            "label": prop.label,
                            "description": prop.description,
                            "dataType": prop.dataType,
                        }
                    )

                # Add properties to the object type in formatted_dict
                formatted_dict[perm_id]["properties"] = properties
            else:
                # If no properties, add an empty dictionary
                formatted_dict[perm_id]["properties"] = {}

    def get_property_dict(self) -> dict:
        """
        Get the property types from openBIS and return them as a dictionary where the keys
        are the property type `code` and the value is a dictionary of attributes assigned to that
        property type.

        Returns:
            dict: Dictionary of property types with their attributes.
        """
        formatted_dict = self._get_formatted_dict("property_types")

        # We return the sorted dictionary in order to have a consistent order for inheritance
        return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

    def get_collection_dict(self) -> dict:
        """
        Get the collection types from openBIS and return them as a dictionary where the keys
        are the collection type `code` and the value is a dictionary of attributes assigned to that
        collection type.

        Returns:
            dict: Dictionary of collection types with their attributes.
        """
        formatted_dict = self._get_formatted_dict("collection_types")
        self._assign_properties(
            entity_name="collection_types", formatted_dict=formatted_dict
        )

        # We return the sorted dictionary in order to have a consistent order for inheritance
        return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

    def get_dataset_dict(self) -> dict:
        """
        Get the dataset types from openBIS and return them as a dictionary where the keys
        are the dataset type `code` and the value is a dictionary of attributes assigned to that
        dataset type.

        Returns:
            dict: Dictionary of dataset types with their attributes.
        """
        formatted_dict = self._get_formatted_dict("dataset_types")
        self._assign_properties(
            entity_name="dataset_types", formatted_dict=formatted_dict
        )

        # We return the sorted dictionary in order to have a consistent order for inheritance
        return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

    def get_object_dict(self) -> dict:
        """
        Get the object types from openBIS and return them as a dictionary where the keys
        are the object type `code` and the value is a dictionary of attributes assigned to that
        object type.

        Returns:
            dict: Dictionary of object types with their attributes.
        """
        formatted_dict = self._get_formatted_dict("object_types")
        self._assign_properties(
            entity_name="object_types", formatted_dict=formatted_dict
        )

        # We return the sorted dictionary in order to have a consistent order for inheritance
        return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

    def get_vocabulary_dict(self) -> dict:
        """
        Get the vocabulary types from openBIS and return them as a dictionary where the keys
        are the vocabulary type `code` and the value is a dictionary of attributes assigned to that
        vocabulary type.

        Returns:
            dict: Dictionary of vocabulary types with their attributes.
        """
        formatted_dict = self._get_formatted_dict("vocabularies")

        # Add properties to each object type
        for voc in self.openbis.get_vocabularies():
            code = voc.code  # Unique identifier for the object type

            # BAM_FLOOR, BAM_HOUSE, BAM_LOCATION, BAM_LOCATION_COMPLETE, BAM_OE, BAM_ROOM, PERSON_STATUS
            # are not exported due to containing sensitive information
            if code in [
                "BAM_FLOOR",
                "BAM_HOUSE",
                "BAM_LOCATION",
                "BAM_LOCATION_COMPLETE",
                "BAM_OE",
                "BAM_ROOM",
                "PERSON_STATUS",
            ]:
                continue
            terms = voc.get_terms()

            if terms:
                # Convert property assignments to list of dictionaries
                terms_dict = terms.df.to_dict(orient="records")

                # Create a dictionary of properties using the correct permId
                voc_terms = {}
                for entry in terms_dict:
                    term_code = entry.get("code", {})
                    if term_code:
                        # Include the desired property fields
                        voc_terms[term_code] = {
                            "code": term_code,
                            "description": entry.get("description", ""),
                            "label": entry.get("label", ""),
                        }

                # Add properties to the object type in formatted_dict
                formatted_dict[code]["terms"] = voc_terms
            else:
                # If no properties, add an empty dictionary
                formatted_dict[code]["terms"] = {}

        # We return the sorted dictionary in order to have a consistent order for inheritance
        return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

get_collection_dict()

Get the collection types from openBIS and return them as a dictionary where the keys are the collection type code and the value is a dictionary of attributes assigned to that collection type.

Returns:

Name Type Description
dict dict

Dictionary of collection types with their attributes.

Source code in bam_masterdata/openbis/get_entities.py
def get_collection_dict(self) -> dict:
    """
    Get the collection types from openBIS and return them as a dictionary where the keys
    are the collection type `code` and the value is a dictionary of attributes assigned to that
    collection type.

    Returns:
        dict: Dictionary of collection types with their attributes.
    """
    formatted_dict = self._get_formatted_dict("collection_types")
    self._assign_properties(
        entity_name="collection_types", formatted_dict=formatted_dict
    )

    # We return the sorted dictionary in order to have a consistent order for inheritance
    return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

get_dataset_dict()

Get the dataset types from openBIS and return them as a dictionary where the keys are the dataset type code and the value is a dictionary of attributes assigned to that dataset type.

Returns:

Name Type Description
dict dict

Dictionary of dataset types with their attributes.

Source code in bam_masterdata/openbis/get_entities.py
def get_dataset_dict(self) -> dict:
    """
    Get the dataset types from openBIS and return them as a dictionary where the keys
    are the dataset type `code` and the value is a dictionary of attributes assigned to that
    dataset type.

    Returns:
        dict: Dictionary of dataset types with their attributes.
    """
    formatted_dict = self._get_formatted_dict("dataset_types")
    self._assign_properties(
        entity_name="dataset_types", formatted_dict=formatted_dict
    )

    # We return the sorted dictionary in order to have a consistent order for inheritance
    return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

get_object_dict()

Get the object types from openBIS and return them as a dictionary where the keys are the object type code and the value is a dictionary of attributes assigned to that object type.

Returns:

Name Type Description
dict dict

Dictionary of object types with their attributes.

Source code in bam_masterdata/openbis/get_entities.py
def get_object_dict(self) -> dict:
    """
    Get the object types from openBIS and return them as a dictionary where the keys
    are the object type `code` and the value is a dictionary of attributes assigned to that
    object type.

    Returns:
        dict: Dictionary of object types with their attributes.
    """
    formatted_dict = self._get_formatted_dict("object_types")
    self._assign_properties(
        entity_name="object_types", formatted_dict=formatted_dict
    )

    # We return the sorted dictionary in order to have a consistent order for inheritance
    return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

get_property_dict()

Get the property types from openBIS and return them as a dictionary where the keys are the property type code and the value is a dictionary of attributes assigned to that property type.

Returns:

Name Type Description
dict dict

Dictionary of property types with their attributes.

Source code in bam_masterdata/openbis/get_entities.py
def get_property_dict(self) -> dict:
    """
    Get the property types from openBIS and return them as a dictionary where the keys
    are the property type `code` and the value is a dictionary of attributes assigned to that
    property type.

    Returns:
        dict: Dictionary of property types with their attributes.
    """
    formatted_dict = self._get_formatted_dict("property_types")

    # We return the sorted dictionary in order to have a consistent order for inheritance
    return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

get_vocabulary_dict()

Get the vocabulary types from openBIS and return them as a dictionary where the keys are the vocabulary type code and the value is a dictionary of attributes assigned to that vocabulary type.

Returns:

Name Type Description
dict dict

Dictionary of vocabulary types with their attributes.

Source code in bam_masterdata/openbis/get_entities.py
def get_vocabulary_dict(self) -> dict:
    """
    Get the vocabulary types from openBIS and return them as a dictionary where the keys
    are the vocabulary type `code` and the value is a dictionary of attributes assigned to that
    vocabulary type.

    Returns:
        dict: Dictionary of vocabulary types with their attributes.
    """
    formatted_dict = self._get_formatted_dict("vocabularies")

    # Add properties to each object type
    for voc in self.openbis.get_vocabularies():
        code = voc.code  # Unique identifier for the object type

        # BAM_FLOOR, BAM_HOUSE, BAM_LOCATION, BAM_LOCATION_COMPLETE, BAM_OE, BAM_ROOM, PERSON_STATUS
        # are not exported due to containing sensitive information
        if code in [
            "BAM_FLOOR",
            "BAM_HOUSE",
            "BAM_LOCATION",
            "BAM_LOCATION_COMPLETE",
            "BAM_OE",
            "BAM_ROOM",
            "PERSON_STATUS",
        ]:
            continue
        terms = voc.get_terms()

        if terms:
            # Convert property assignments to list of dictionaries
            terms_dict = terms.df.to_dict(orient="records")

            # Create a dictionary of properties using the correct permId
            voc_terms = {}
            for entry in terms_dict:
                term_code = entry.get("code", {})
                if term_code:
                    # Include the desired property fields
                    voc_terms[term_code] = {
                        "code": term_code,
                        "description": entry.get("description", ""),
                        "label": entry.get("label", ""),
                    }

            # Add properties to the object type in formatted_dict
            formatted_dict[code]["terms"] = voc_terms
        else:
            # If no properties, add an empty dictionary
            formatted_dict[code]["terms"] = {}

    # We return the sorted dictionary in order to have a consistent order for inheritance
    return dict(sorted(formatted_dict.items(), key=lambda item: item[0].count(".")))

bam_masterdata.checker.checker

MasterdataChecker

Source code in bam_masterdata/checker/checker.py
class MasterdataChecker:
    VALID_MODES = {"self", "incoming", "validate", "compare", "all", "individual"}

    def __init__(self):
        """
        Initialize the comparator with validation rules and set the datamodel directory.
        """
        self.current_model: dict = None
        self.new_entities: dict = None
        self.logger = logger
        self.validation_rules: dict = {}

    def load_current_model(self, datamodel_dir: str = "./bam_masterdata/datamodel/"):
        """
        Load and transform the current data model (Pydantic classes) into JSON.

        Uses the default datamodel directory unless overridden.
        """
        self.logger.info(f"Loading current data model from: {datamodel_dir}")
        entities_dict = EntitiesDict(python_path=datamodel_dir, logger=self.logger)
        self.current_model = entities_dict.single_json()

    def load_new_entities(self, source: str):
        """
        Load new entities from various sources (Python classes, Excel, etc.).
        """
        self.logger.info(f"Loading new entities from: {source}")
        loader = SourceLoader(source)
        self.new_entities = loader.load()

    def check(self, mode: str = "all") -> dict:
        """
        Run validations.

        Modes:
        - "self" -> Validate only the current data model.
        - "incoming" -> Validate only the new entity structure.
        - "validate" -> Validate both the current model and new entities.
        - "compare" -> Compare new entities against the current model.
        - "all" -> Run both validation types.
        - "individual" -> Run individual repositories validations.

        Before running, ensure that required models are loaded based on the mode.

        Returns:
            dict: Validation results.
        """
        # Validate mode selection
        if mode not in self.VALID_MODES:
            raise ValueError(f"Invalid mode: {mode}. Choose from {self.VALID_MODES}.")

        # Load required models based on the selected mode
        if (
            mode in ["self", "validate", "compare", "all", "individual"]
            and self.current_model is None
        ):
            self.logger.info("Current model is missing. Loading it from local files.")
            self.load_current_model()

        if (
            mode in ["incoming", "validate", "compare", "all", "individual"]
            and self.new_entities is None
        ):
            raise ValueError(
                "New entities must be loaded before validation in 'incoming', 'validate', 'individual', 'compare', or 'all' modes."
            )

        # Load the validation rules
        if (
            mode in ["self", "incoming", "validate", "all", "individual"]
            and self.validation_rules == {}
        ):
            self.validation_rules = load_validation_rules(self.logger)

        validator = MasterdataValidator(
            self.new_entities, self.current_model, self.validation_rules
        )
        return validator.validate(mode)

__init__()

Initialize the comparator with validation rules and set the datamodel directory.

Source code in bam_masterdata/checker/checker.py
def __init__(self):
    """
    Initialize the comparator with validation rules and set the datamodel directory.
    """
    self.current_model: dict = None
    self.new_entities: dict = None
    self.logger = logger
    self.validation_rules: dict = {}

check(mode='all')

Run validations.

Modes: - "self" -> Validate only the current data model. - "incoming" -> Validate only the new entity structure. - "validate" -> Validate both the current model and new entities. - "compare" -> Compare new entities against the current model. - "all" -> Run both validation types. - "individual" -> Run individual repositories validations.

Before running, ensure that required models are loaded based on the mode.

Returns:

Name Type Description
dict dict

Validation results.

Source code in bam_masterdata/checker/checker.py
def check(self, mode: str = "all") -> dict:
    """
    Run validations.

    Modes:
    - "self" -> Validate only the current data model.
    - "incoming" -> Validate only the new entity structure.
    - "validate" -> Validate both the current model and new entities.
    - "compare" -> Compare new entities against the current model.
    - "all" -> Run both validation types.
    - "individual" -> Run individual repositories validations.

    Before running, ensure that required models are loaded based on the mode.

    Returns:
        dict: Validation results.
    """
    # Validate mode selection
    if mode not in self.VALID_MODES:
        raise ValueError(f"Invalid mode: {mode}. Choose from {self.VALID_MODES}.")

    # Load required models based on the selected mode
    if (
        mode in ["self", "validate", "compare", "all", "individual"]
        and self.current_model is None
    ):
        self.logger.info("Current model is missing. Loading it from local files.")
        self.load_current_model()

    if (
        mode in ["incoming", "validate", "compare", "all", "individual"]
        and self.new_entities is None
    ):
        raise ValueError(
            "New entities must be loaded before validation in 'incoming', 'validate', 'individual', 'compare', or 'all' modes."
        )

    # Load the validation rules
    if (
        mode in ["self", "incoming", "validate", "all", "individual"]
        and self.validation_rules == {}
    ):
        self.validation_rules = load_validation_rules(self.logger)

    validator = MasterdataValidator(
        self.new_entities, self.current_model, self.validation_rules
    )
    return validator.validate(mode)

load_current_model(datamodel_dir='./bam_masterdata/datamodel/')

Load and transform the current data model (Pydantic classes) into JSON.

Uses the default datamodel directory unless overridden.

Source code in bam_masterdata/checker/checker.py
def load_current_model(self, datamodel_dir: str = "./bam_masterdata/datamodel/"):
    """
    Load and transform the current data model (Pydantic classes) into JSON.

    Uses the default datamodel directory unless overridden.
    """
    self.logger.info(f"Loading current data model from: {datamodel_dir}")
    entities_dict = EntitiesDict(python_path=datamodel_dir, logger=self.logger)
    self.current_model = entities_dict.single_json()

load_new_entities(source)

Load new entities from various sources (Python classes, Excel, etc.).

Source code in bam_masterdata/checker/checker.py
def load_new_entities(self, source: str):
    """
    Load new entities from various sources (Python classes, Excel, etc.).
    """
    self.logger.info(f"Loading new entities from: {source}")
    loader = SourceLoader(source)
    self.new_entities = loader.load()

bam_masterdata.checker.masterdata_validator

MasterdataValidator

Source code in bam_masterdata/checker/masterdata_validator.py
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
class MasterdataValidator:
    def __init__(self, new_entities: dict, current_model: dict, validation_rules: dict):
        """
        Initialize the validator with new and current entity data.

        Args:
            new_entities (dict): The incoming datamodel.
            current_model (dict): The existing datamodel.
            validation_rules (dict): The validation rules to apply.
        """
        self.new_entities = new_entities
        self.current_model = current_model
        self.validation_rules = validation_rules
        self.logger = logger
        self.log_msgs: list = []
        self.validation_results: dict = {}

    def validate(self, mode: str = "all") -> dict:
        """
        Run validations based on mode:
        - "self": Validate current model structure and format.
        - "incoming": Validate new entities structure and format.
        - "validate": Validate both current and incoming models but do not compare.
        - "compare": Validate new entities against the current model.
        - "all": Run both.
        - "individual": Validate new entities and compare them with the current model.

         Returns:
            dict: Validation results.
        """
        self.logger.debug("Starting validation process...", mode=mode)

        # Reset validation results before running checks
        self.validation_results = {
            "current_model": {},
            "incoming_model": {},
            "comparisons": {},
        }

        if mode in ["self", "all", "validate"]:
            self.logger.debug("Validating current model...")
            self._validate_model(self.current_model)
            self._extract_log_messages(
                self.current_model, self.validation_results["current_model"]
            )

        if mode in ["incoming", "all", "validate"]:
            self.logger.debug("Validating new entities...")
            self._validate_model(self.new_entities)
            self._extract_log_messages(
                self.new_entities, self.validation_results["incoming_model"]
            )

        if mode in ["compare", "all"]:
            self.logger.debug("Comparing new entities with current model...")
            self._compare_with_current_model(mode=mode)
            self._extract_log_messages(
                self.new_entities, self.validation_results["comparisons"]
            )

        if mode == "individual":
            self.logger.debug(
                "Validating new entities and comparing them with current model..."
            )
            self.validation_results = {
                "incoming_model": {},
                "comparisons": {},
            }
            self._validate_model(self.new_entities)
            self._extract_log_messages(
                self.new_entities, self.validation_results["incoming_model"]
            )
            self._compare_with_current_model(mode="individual")
            self._extract_log_messages(
                self.new_entities, self.validation_results["comparisons"]
            )

        return self.validation_results

    def _validate_model(self, model: dict) -> dict:
        """
        Validate the given datamodel against the validation rules.

        Args:
            model (dict): The datamodel to validate.

        Returns:
            dict: A dictionary containing ...
        """
        for entity_type, entities in model.items():
            for entity_name, entity_data in entities.items():
                entity_id = entity_data.get("defs", {}).get("code", entity_name)

                # Ensure _log_msgs exists
                if "_log_msgs" not in entity_data:
                    entity_data["_log_msgs"] = []

                if model == self.new_entities:
                    self.logger.info(f"Validating {entity_type} -> {entity_id}")

                # Validate 'defs'
                if "defs" in entity_data:
                    row_location = entity_data["defs"].get("row_location", "Unknown")
                    self._validate_fields(
                        entity_data["defs"],
                        "defs_validation",
                        entity_type,
                        entity_id,
                        row_location,
                        entity_data,
                    )

                # Collect ordered sections for each entity
                entity_sections = []
                # Validate 'properties' (except for vocabulary_types, which uses 'terms')
                if (
                    entity_type != "vocabulary_types"
                    and entity_type != "vocabulary_type"
                ) and "properties" in entity_data:
                    for prop in entity_data["properties"]:
                        row_location = prop.get("row_location", "Unknown")

                        # Collect section names in order
                        section = prop.get("section", "").strip()
                        if section:  # Avoid empty sections
                            entity_sections.append(
                                {
                                    "code": prop["code"],
                                    "section": section,
                                    "row_location": row_location,
                                }
                            )

                        # Check for deprecated `$ANNOTATIONS_STATE`
                        if (
                            prop["code"] == "$ANNOTATIONS_STATE"
                            and model == self.new_entities
                        ):
                            log_message = (
                                f"Property $ANNOTATIONS_STATE is deprecated from openBIS 20.10.7.3. "
                                f"Assigned to entity '{entity_id}' at row {row_location}."
                            )
                            store_log_message(
                                logger, entity_data, log_message, level="warning"
                            )

                        self._validate_fields(
                            prop,
                            "properties_validation",
                            entity_type,
                            entity_id,
                            row_location,
                            entity_data,
                        )

                # TODO: revise if these checks about ordering of sections are truly necessary
                # Check if "Additional Information" is followed only by "Additional Information" or "Comments"
                for i in range(len(entity_sections) - 1):
                    current_section = entity_sections[i]["section"]
                    next_section = entity_sections[i + 1]["section"]
                    row_location = entity_sections[i + 1]["row_location"]

                    if (
                        current_section == "Additional Information"
                        and next_section not in {"Additional Information", "Comments"}
                    ):
                        log_message = (
                            f"Invalid section order: 'Additional Information' at row {entity_sections[i]['row_location']} "
                            f"must be followed by 'Comments', but found '{next_section}' at row {row_location}."
                        )
                        store_log_message(
                            logger, entity_data, log_message, level="error"
                        )

                # Check if required properties exist in specific sections
                required_properties = {
                    "Additional Information": ["notes"],
                    "Comments": ["comments", "xmlcomments", "$xmlcomments"],
                }

                # Track found properties
                found_properties = {section: False for section in required_properties}

                for entry in entity_sections:
                    section = entry["section"]
                    property_code = entry["code"]
                    row_location = entry["row_location"]

                    # Check if this section is one we need to validate
                    if section in required_properties:
                        # Perform a case-insensitive check against the list of allowed property codes
                        if property_code.lower() in required_properties[section]:
                            found_properties[section] = True

                # Log errors for missing required properties
                for section, prop in required_properties.items():
                    if (
                        any(entry["section"] == section for entry in entity_sections)
                        and not found_properties[section]
                    ):
                        log_message = f"Missing required property '{prop}' in section '{section}'."
                        store_log_message(
                            logger, entity_data, log_message, level="error"
                        )

                # Validate 'terms' (only for vocabulary_types)
                if (
                    entity_type in ["vocabulary_types", "vocabulary_type"]
                    and "terms" in entity_data
                ):
                    for term in entity_data["terms"]:
                        row_location = term.get("row_location", "Unknown")
                        self._validate_fields(
                            term,
                            "terms_validation",
                            entity_type,
                            entity_id,
                            row_location,
                            entity_data,
                        )

        return entity_data

    def _validate_fields(
        self,
        data: dict,
        rule_type: str,
        entity_type: str,
        entity_name: str,
        row_location: str,
        parent_entity: dict,
    ):
        """
        Validate a dictionary of fields against the corresponding validation rules.

        Args:
            data (dict): The fields to validate.
            rule_type (str): The rule section to use ("defs_validation", "properties_validation", or "terms_validation").
            entity_type (str): The entity type being validated.
            entity_name (str): The specific entity name (ID if available).
            row_location (str): The row where the entity is located in the source file.
            parent_entity (dict): The entity dictionary where _log_msgs should be stored.
        """

        # Determine where the issue is occurring (in properties, terms, or main entity fields)
        extra_location = {
            "properties_validation": "in 'properties'.",
            "terms_validation": "in 'terms'.",
        }.get(rule_type, ".")

        for field, value in data.items():
            rule = self.validation_rules.get(rule_type, {}).get(field)

            extra_location_str = f" {extra_location} " if extra_location else " "

            log_message = (
                f"Invalid '{value}' value found in the '{field}' field at line {row_location} "
                f"in entity '{entity_name}' of '{entity_type}'{extra_location_str}"
            )

            if not rule:
                continue  # Skip fields with no validation rules

            # Handle empty fields
            if "allow_empty" in rule and (value is None or value == "" or not value):
                continue  # Skip check if empty fields are allowed

            # Validate pattern (regex)
            if "pattern" in rule and value is not None:
                if not re.match(rule["pattern"], str(value)):
                    log_message = f"{log_message}Invalid format."
                    level = "error"
                    if "is_description" in rule:
                        log_message = f"{log_message} Description should follow the schema: English Description + '//' + German Description. "
                        level = "warning"
                    if "is_section" in rule:
                        log_message = f"{log_message} First letter of every word starts with capitalized lettter."
                        level = "warning"
                    store_log_message(logger, parent_entity, log_message, level=level)

            # Validate boolean fields
            if "is_bool" in rule and str(value).strip().lower() not in [
                "true",
                "false",
            ]:
                store_log_message(
                    logger,
                    parent_entity,
                    f"{log_message}Expected a boolean.",
                    level="error",
                )

            # Validate data types
            if "is_data" in rule:
                str_val = str(value)
                is_valid_standard = str_val in [dt.value for dt in DataType]
                is_valid_dynamic = False

                if not is_valid_standard and (
                    str_val.startswith("OBJECT") or str_val.startswith("SAMPLE")
                ):
                    els = str_val.split(":")

                    if len(els) == 2 and els[1].strip():
                        is_valid_dynamic = True

                if not is_valid_standard and not is_valid_dynamic:
                    store_log_message(
                        logger,
                        parent_entity,
                        f"{log_message}The Data Type should be one of the following: {[dt.value for dt in DataType]} or follow the format 'SAMPLE:<CODE>' or 'OBJECT:<CODE>'",
                        level="error",
                    )

            # Validate special cases (e.g., extra validation functions)
            if "extra_validation" in rule:
                validation_func = getattr(self, rule["extra_validation"], None)
                if validation_func == "is_reduced_version" and not is_reduced_version(
                    value, entity_name
                ):
                    store_log_message(
                        logger,
                        parent_entity,
                        f"{log_message}The generated code should be a part of the code.",
                        level="warning",
                    )

    def _compare_with_current_model(self, mode) -> dict:
        """
        Compare new entities against the current model using validation rules.
        """
        self.logger.debug("Starting comparison with the current model...")

        new_entity = False

        all_props = self.extract_property_codes(self.current_model)

        for entity_type, incoming_entities in self.new_entities.items():
            if entity_type not in self.current_model:
                continue  # Skip if entity type does not exist in the current model

            current_entities = self.current_model[entity_type]

            for entity_code, incoming_entity in incoming_entities.items():
                incoming_row_location = "Unknown"
                current_entity = current_entities.get(entity_code)

                # Ensure _log_msgs exists
                if "_log_msgs" not in incoming_entity:
                    incoming_entity["_log_msgs"] = []

                if current_entity:
                    if mode == "individual":
                        log_message = f"The entity {entity_code} already exists in `bam-masterdata`. Please, check your classes. "
                        store_log_message(
                            logger, incoming_entity, log_message, level="critical"
                        )
                    # Compare general attributes for all entities
                    for key, new_value in incoming_entity.get("defs", {}).items():
                        incoming_row_location = incoming_entity.get("defs", {}).get(
                            "row_location", "Unknown"
                        )
                        old_value = current_entity.get("defs", {}).get(key)
                        if (
                            (key != "code" and key != "row_location")
                            and old_value is not None
                            and new_value != old_value
                        ):
                            log_message = (
                                f"Entity type {entity_code} has changed its attribute {key} "
                                f"from '{old_value}' to '{new_value}' at row {incoming_row_location}."
                            )
                            store_log_message(
                                logger, incoming_entity, log_message, level="warning"
                            )

                    # Special case for `property_types`
                    if entity_type in ("property_types", "property_type"):
                        incoming_row_location = incoming_entity.get(
                            "row_location", "Unknown"
                        )
                        new_data_type = incoming_entity.get("data_type")
                        old_data_type = current_entity.get("data_type")

                        if (
                            new_data_type
                            and old_data_type
                            and new_data_type != old_data_type
                        ):
                            log_message = (
                                f"Property type {entity_code} has changed its `data_type` value from {old_data_type} to {new_data_type} at row {incoming_row_location}. "
                                "This will cause that data using the Property with inconsistent versions of data type will probably break openBIS. "
                                "You need to define a new property with the new data type or revise your data model."
                            )
                            store_log_message(
                                logger, incoming_entity, log_message, level="critical"
                            )

                        if (
                            new_data_type == "CONTROLLEDVOCABULARY"
                            and incoming_entity.get("vocabulary_code")
                            != current_entity.get("vocabulary_code")
                        ):
                            old_vocabulary = current_entity.get("vocabulary_code")
                            new_vocabulary = incoming_entity.get("vocabulary_code")
                            log_message = (
                                f"Property type {entity_code} using controlled vocabulary has changed its `vocabulary_code` value from {old_vocabulary} to {new_vocabulary}, "
                                f"at row {incoming_row_location} which means that data using a type that is not compatible with the new type will probably break openBIS. "
                                "You need to define a new property with the new data type or revise your data model."
                            )
                            store_log_message(
                                logger, incoming_entity, log_message, level="critical"
                            )

                else:
                    new_entity = True

                # Compare assigned properties or terms
                if "properties" in incoming_entity:
                    self._compare_assigned_properties(
                        entity_code,
                        incoming_entity,
                        current_entity,
                        entity_type,
                        new_entity,
                        incoming_row_location,
                        all_props,
                    )
                elif "terms" in incoming_entity:
                    self._compare_assigned_properties(
                        entity_code,
                        incoming_entity,
                        current_entity,
                        entity_type,
                        new_entity,
                        incoming_row_location,
                        all_props,
                        is_terms=True,
                    )

        if not self.validation_results.get("comparisons"):
            logger.info(
                "No critical conflicts found between new entities compared to the current model."
            )

        return self.validation_results

    def _compare_assigned_properties(
        self,
        entity_code,
        incoming_entity,
        current_entity,
        entity_type,
        new_entity,
        incoming_row_location,
        all_props,
        is_terms=False,
    ):
        """
        Compares assigned properties (for ObjectType, CollectionType, etc.) or terms (for VocabularyType).
        """
        incoming_props = {
            prop["code"]: prop
            for prop in incoming_entity.get(
                "properties" if not is_terms else "terms", []
            )
        }

        incoming_prop_codes = set(incoming_props.keys())

        if not new_entity:
            current_props = {
                prop["code"]: prop
                for prop in current_entity.get(
                    "properties" if not is_terms else "terms", []
                )
            }

            # Check for non-existing assigned properties
            current_prop_codes = set(current_props.keys())

            for prop_code in incoming_prop_codes:
                if prop_code not in all_props and is_terms is False:
                    log_message = (
                        f"The assigned property {prop_code} to the entity {entity_code} at row {incoming_props[prop_code].get('row_location')} does not exist in openBIS. "
                        "Please, define it in your PropertyType section."
                    )
                    store_log_message(
                        logger, incoming_entity, log_message, level="error"
                    )

            # Check for existing changes in assigned properties
            missing_properties = incoming_prop_codes - current_prop_codes
            deleted_properties = current_prop_codes - incoming_prop_codes

            if missing_properties or deleted_properties:
                log_message = f"The assigned properties to {entity_code} at row {incoming_row_location} have changed:"
                store_log_message(logger, incoming_entity, log_message, level="warning")

            # Check for missing properties
            for missing in missing_properties:
                log_message = f"{missing} has been added as a new property at row {incoming_props[missing].get('row_location')}."
                store_log_message(logger, incoming_entity, log_message, level="info")

            # Check for deleted properties
            for deleted in deleted_properties:
                log_message = f"{deleted} has been deleted."
                store_log_message(logger, incoming_entity, log_message, level="warning")

            # Check for property modifications
            common_props = incoming_prop_codes & current_prop_codes
            for prop_code in common_props:
                new_prop = incoming_props[prop_code]
                old_prop = current_props[prop_code]

                for key, new_value in new_prop.items():
                    old_value = old_prop.get(key)
                    if (
                        (key != "code" and key != "row_location")
                        and old_value is not None
                        and new_value != old_value
                    ):
                        log_message = (
                            f"Assigned property {prop_code} to entity type {entity_code} has changed its attribute {key} "
                            f"from '{old_value}' to '{new_value}' at row {incoming_props[prop_code].get('row_location')}."
                        )
                        store_log_message(
                            logger, incoming_entity, log_message, level="warning"
                        )

        # Check if assigned properties match another entity's properties
        for other_entity_code, other_entity in self.current_model.get(
            entity_type, {}
        ).items():
            if other_entity_code != entity_code:
                other_entity_properties = (
                    other_entity.get("properties", [])
                    if not is_terms
                    else other_entity.get("terms", [])
                )
                other_entity_props = {prop["code"] for prop in other_entity_properties}

                if (incoming_prop_codes == other_entity_props) and incoming_prop_codes:
                    log_message = (
                        "Entity will not be imported in openBIS. "
                        f"The entity {entity_code} at row {incoming_entity['defs'].get('row_location')} has the same properties defined as {other_entity_code}. "
                        "Maybe they are representing the same entity?"
                    )
                    store_log_message(
                        logger, incoming_entity, log_message, level="warning"
                    )

    def _extract_log_messages(self, model: dict, target_dict: dict) -> None:
        """
        Extracts and appends _log_msgs from the validated entities into an existing dictionary.

        Args:
            model (dict): The validated entity model.
            target_dict (dict): The dictionary where logs should be appended.
        """
        for entity_type, entities in model.items():
            if entity_type not in target_dict:
                target_dict[entity_type] = {}

            for entity_name, entity_data in entities.items():
                if "_log_msgs" in entity_data and entity_data["_log_msgs"]:
                    if entity_name not in target_dict[entity_type]:
                        target_dict[entity_type][entity_name] = {"_log_msgs": []}

                    # Append new messages to the existing ones
                    target_dict[entity_type][entity_name]["_log_msgs"].extend(
                        entity_data["_log_msgs"]
                    )

    def extract_property_codes(self, data):
        codes = set()

        # Check if the data contains 'properties' and extract 'code'
        if isinstance(data, dict):
            for key, value in data.items():
                # If the key is 'properties', collect all the 'code' values
                if key == "properties" and isinstance(value, list):
                    for property_item in value:
                        if "code" in property_item:
                            codes.add(property_item["code"])
                # Recursively check for more nested structures
                elif isinstance(value, dict | list):
                    codes.update(self.extract_property_codes(value))

        elif isinstance(data, list):
            for item in data:
                codes.update(self.extract_property_codes(item))

        return codes

__init__(new_entities, current_model, validation_rules)

Initialize the validator with new and current entity data.

Parameters:

Name Type Description Default
new_entities dict

The incoming datamodel.

required
current_model dict

The existing datamodel.

required
validation_rules dict

The validation rules to apply.

required
Source code in bam_masterdata/checker/masterdata_validator.py
def __init__(self, new_entities: dict, current_model: dict, validation_rules: dict):
    """
    Initialize the validator with new and current entity data.

    Args:
        new_entities (dict): The incoming datamodel.
        current_model (dict): The existing datamodel.
        validation_rules (dict): The validation rules to apply.
    """
    self.new_entities = new_entities
    self.current_model = current_model
    self.validation_rules = validation_rules
    self.logger = logger
    self.log_msgs: list = []
    self.validation_results: dict = {}

validate(mode='all')

Run validations based on mode: - "self": Validate current model structure and format. - "incoming": Validate new entities structure and format. - "validate": Validate both current and incoming models but do not compare. - "compare": Validate new entities against the current model. - "all": Run both. - "individual": Validate new entities and compare them with the current model.

Returns: dict: Validation results.

Source code in bam_masterdata/checker/masterdata_validator.py
def validate(self, mode: str = "all") -> dict:
    """
    Run validations based on mode:
    - "self": Validate current model structure and format.
    - "incoming": Validate new entities structure and format.
    - "validate": Validate both current and incoming models but do not compare.
    - "compare": Validate new entities against the current model.
    - "all": Run both.
    - "individual": Validate new entities and compare them with the current model.

     Returns:
        dict: Validation results.
    """
    self.logger.debug("Starting validation process...", mode=mode)

    # Reset validation results before running checks
    self.validation_results = {
        "current_model": {},
        "incoming_model": {},
        "comparisons": {},
    }

    if mode in ["self", "all", "validate"]:
        self.logger.debug("Validating current model...")
        self._validate_model(self.current_model)
        self._extract_log_messages(
            self.current_model, self.validation_results["current_model"]
        )

    if mode in ["incoming", "all", "validate"]:
        self.logger.debug("Validating new entities...")
        self._validate_model(self.new_entities)
        self._extract_log_messages(
            self.new_entities, self.validation_results["incoming_model"]
        )

    if mode in ["compare", "all"]:
        self.logger.debug("Comparing new entities with current model...")
        self._compare_with_current_model(mode=mode)
        self._extract_log_messages(
            self.new_entities, self.validation_results["comparisons"]
        )

    if mode == "individual":
        self.logger.debug(
            "Validating new entities and comparing them with current model..."
        )
        self.validation_results = {
            "incoming_model": {},
            "comparisons": {},
        }
        self._validate_model(self.new_entities)
        self._extract_log_messages(
            self.new_entities, self.validation_results["incoming_model"]
        )
        self._compare_with_current_model(mode="individual")
        self._extract_log_messages(
            self.new_entities, self.validation_results["comparisons"]
        )

    return self.validation_results

bam_masterdata.checker.source_loader

SourceLoader

Load the entities from a source written in different formats (Python classes, Excel, etc.) as defined in the source_path into a dictionary.

Source code in bam_masterdata/checker/source_loader.py
class SourceLoader:
    """
    Load the entities from a source written in different formats (Python classes, Excel, etc.) as defined
    in the `source_path` into a dictionary.
    """

    def __init__(self, source_path: str, **kwargs):
        self.source_path = source_path
        self.logger = kwargs.get("logger", logger)
        self.row_cell_info = kwargs.get("row_cell_info", True)
        # Check if the path is a single .py file OR a directory containing .py files
        if self.source_path.endswith(".py") or (
            os.path.isdir(self.source_path)
            and any(glob.glob(os.path.join(self.source_path, "*.py")))
        ):
            self.source_type = "python"
        elif self.source_path.endswith(".xlsx"):
            self.source_type = "excel"
        else:
            self.source_type = None
            self.logger.warning(f"Unsupported source type for path: {source_path}")

    def load(self) -> dict:
        """
        Load entities from the source path into a dictionary.

        Returns:
            dict: A dictionary containing the entities.
        """
        self.logger.info(f"Source type: {self.source_type}")
        if self.source_type == "python":
            return convert_enums(
                EntitiesDict(python_path=self.source_path).single_json()
            )
        elif self.source_type == "excel":
            return self.entities_to_json()
        else:
            raise NotImplementedError(f"Source type {self.source_type} not supported.")

    def entities_to_json(self) -> dict:
        """
        Transforms the dictionary of entities returned by the Excel extractor into a dictionary in JSON format for later check.

        Returns:
            dict: A dictionary containing the transformed entities.
        """

        excel_entities = MasterdataExcelExtractor(
            excel_path=self.source_path, row_cell_info=self.row_cell_info
        ).excel_to_entities()

        transformed_data = {}

        for entity_type, entities in excel_entities.items():
            transformed_data[entity_type] = {}

            for entity_name, entity_data in entities.items():
                if entity_type in ("vocabulary_type", "vocabulary_types"):
                    transformed_entity = {
                        "terms": [],  # Now placed before "defs"
                        "defs": {  # Metadata moved to the end
                            "code": entity_data.get("code"),
                            "description": entity_data.get("description", ""),
                            "id": format_json_id(
                                entity_name
                            ),  # PascalCase for entity ID
                            "row_location": entity_data.get("row_location"),
                            "url_template": entity_data.get("url_template") or None,
                        },
                    }
                else:
                    transformed_entity = {
                        "properties": [],  # Now placed before "defs"
                        "defs": {  # Metadata moved to the end
                            "code": entity_data.get("code"),
                            "description": entity_data.get("description", ""),
                            "id": format_json_id(
                                entity_name
                            ),  # PascalCase for entity ID
                            "row_location": entity_data.get("row_location"),
                            "validation_script": entity_data.get(
                                "validationPlugin"
                            ).strip()
                            if isinstance(entity_data.get("validationPlugin"), str)
                            else None,
                            "iri": entity_data.get("iri") or None,  # Convert "" to None
                        },
                    }

                # Handle additional fields specific to dataset_types
                if entity_type in ("dataset_types", "dataset_type"):
                    transformed_entity["defs"]["main_dataset_pattern"] = (
                        entity_data.get("main_dataset_pattern")
                    )
                    transformed_entity["defs"]["main_dataset_path"] = entity_data.get(
                        "main_dataset_path"
                    )

                # Handle additional fields specific to object_types
                if entity_type in ("object_types", "object_type"):
                    transformed_entity["defs"]["generated_code_prefix"] = (
                        entity_data.get("generatedCodePrefix")
                    )
                    transformed_entity["defs"]["auto_generate_codes"] = entity_data.get(
                        "autoGeneratedCode"
                    )

                # Convert properties from dict to list
                if "properties" in entity_data:
                    for prop_name, prop_data in entity_data["properties"].items():
                        data_type = prop_data.get("dataType")
                        object_code = prop_data.get("objectCode")
                        if isinstance(data_type, str):
                            data_type_upper = data_type.upper()
                            if ":" in data_type_upper:
                                prefix, dynamic_code = data_type_upper.split(":", 1)
                                if (
                                    prefix in ("SAMPLE", "OBJECT")
                                    and dynamic_code.strip()
                                ):
                                    data_type_upper = "OBJECT"
                                    object_code = object_code or dynamic_code.strip()
                            data_type = data_type_upper

                        transformed_property = {
                            "code": prop_data.get("code"),
                            "description": prop_data.get("description", ""),
                            "id": format_json_id(
                                prop_name
                            ),  # Now correctly formatted to PascalCase
                            "row_location": prop_data.get("row_location"),
                            "iri": prop_data.get("iri") or None,  # Convert "" to None
                            "property_label": prop_data.get("label"),
                            "data_type": data_type,
                            "vocabulary_code": prop_data.get("vocabularyCode")
                            or None,  # Convert "" to None
                            "object_code": object_code,
                            "metadata": None,
                            "dynamic_script": None,
                            "mandatory": prop_data.get("mandatory", False),
                            "show_in_edit_views": prop_data.get(
                                "show_in_edit_views", False
                            ),
                            "section": prop_data.get("section", ""),
                            "unique": None,
                            "internal_assignment": None,
                        }
                        transformed_entity["properties"].append(transformed_property)

                if "terms" in entity_data:
                    transformed_entity.setdefault("terms", [])
                    for term_name, term_data in entity_data.get("terms", "{}").items():
                        transformed_term = {
                            "code": term_data.get("code"),
                            "description": term_data.get("description", ""),
                            "id": format_json_id(
                                term_name
                            ),  # Now correctly formatted to PascalCase
                            "row_location": term_data.get("row_location"),
                            "url_template": term_data.get("url_template")
                            or None,  # Convert "" to None
                            "label": term_data.get("label"),
                            "official": term_data.get("official"),
                        }
                        transformed_entity["terms"].append(transformed_term)

                transformed_data[entity_type][entity_name] = transformed_entity

        return transformed_data

entities_to_json()

Transforms the dictionary of entities returned by the Excel extractor into a dictionary in JSON format for later check.

Returns:

Name Type Description
dict dict

A dictionary containing the transformed entities.

Source code in bam_masterdata/checker/source_loader.py
def entities_to_json(self) -> dict:
    """
    Transforms the dictionary of entities returned by the Excel extractor into a dictionary in JSON format for later check.

    Returns:
        dict: A dictionary containing the transformed entities.
    """

    excel_entities = MasterdataExcelExtractor(
        excel_path=self.source_path, row_cell_info=self.row_cell_info
    ).excel_to_entities()

    transformed_data = {}

    for entity_type, entities in excel_entities.items():
        transformed_data[entity_type] = {}

        for entity_name, entity_data in entities.items():
            if entity_type in ("vocabulary_type", "vocabulary_types"):
                transformed_entity = {
                    "terms": [],  # Now placed before "defs"
                    "defs": {  # Metadata moved to the end
                        "code": entity_data.get("code"),
                        "description": entity_data.get("description", ""),
                        "id": format_json_id(
                            entity_name
                        ),  # PascalCase for entity ID
                        "row_location": entity_data.get("row_location"),
                        "url_template": entity_data.get("url_template") or None,
                    },
                }
            else:
                transformed_entity = {
                    "properties": [],  # Now placed before "defs"
                    "defs": {  # Metadata moved to the end
                        "code": entity_data.get("code"),
                        "description": entity_data.get("description", ""),
                        "id": format_json_id(
                            entity_name
                        ),  # PascalCase for entity ID
                        "row_location": entity_data.get("row_location"),
                        "validation_script": entity_data.get(
                            "validationPlugin"
                        ).strip()
                        if isinstance(entity_data.get("validationPlugin"), str)
                        else None,
                        "iri": entity_data.get("iri") or None,  # Convert "" to None
                    },
                }

            # Handle additional fields specific to dataset_types
            if entity_type in ("dataset_types", "dataset_type"):
                transformed_entity["defs"]["main_dataset_pattern"] = (
                    entity_data.get("main_dataset_pattern")
                )
                transformed_entity["defs"]["main_dataset_path"] = entity_data.get(
                    "main_dataset_path"
                )

            # Handle additional fields specific to object_types
            if entity_type in ("object_types", "object_type"):
                transformed_entity["defs"]["generated_code_prefix"] = (
                    entity_data.get("generatedCodePrefix")
                )
                transformed_entity["defs"]["auto_generate_codes"] = entity_data.get(
                    "autoGeneratedCode"
                )

            # Convert properties from dict to list
            if "properties" in entity_data:
                for prop_name, prop_data in entity_data["properties"].items():
                    data_type = prop_data.get("dataType")
                    object_code = prop_data.get("objectCode")
                    if isinstance(data_type, str):
                        data_type_upper = data_type.upper()
                        if ":" in data_type_upper:
                            prefix, dynamic_code = data_type_upper.split(":", 1)
                            if (
                                prefix in ("SAMPLE", "OBJECT")
                                and dynamic_code.strip()
                            ):
                                data_type_upper = "OBJECT"
                                object_code = object_code or dynamic_code.strip()
                        data_type = data_type_upper

                    transformed_property = {
                        "code": prop_data.get("code"),
                        "description": prop_data.get("description", ""),
                        "id": format_json_id(
                            prop_name
                        ),  # Now correctly formatted to PascalCase
                        "row_location": prop_data.get("row_location"),
                        "iri": prop_data.get("iri") or None,  # Convert "" to None
                        "property_label": prop_data.get("label"),
                        "data_type": data_type,
                        "vocabulary_code": prop_data.get("vocabularyCode")
                        or None,  # Convert "" to None
                        "object_code": object_code,
                        "metadata": None,
                        "dynamic_script": None,
                        "mandatory": prop_data.get("mandatory", False),
                        "show_in_edit_views": prop_data.get(
                            "show_in_edit_views", False
                        ),
                        "section": prop_data.get("section", ""),
                        "unique": None,
                        "internal_assignment": None,
                    }
                    transformed_entity["properties"].append(transformed_property)

            if "terms" in entity_data:
                transformed_entity.setdefault("terms", [])
                for term_name, term_data in entity_data.get("terms", "{}").items():
                    transformed_term = {
                        "code": term_data.get("code"),
                        "description": term_data.get("description", ""),
                        "id": format_json_id(
                            term_name
                        ),  # Now correctly formatted to PascalCase
                        "row_location": term_data.get("row_location"),
                        "url_template": term_data.get("url_template")
                        or None,  # Convert "" to None
                        "label": term_data.get("label"),
                        "official": term_data.get("official"),
                    }
                    transformed_entity["terms"].append(transformed_term)

            transformed_data[entity_type][entity_name] = transformed_entity

    return transformed_data

load()

Load entities from the source path into a dictionary.

Returns:

Name Type Description
dict dict

A dictionary containing the entities.

Source code in bam_masterdata/checker/source_loader.py
def load(self) -> dict:
    """
    Load entities from the source path into a dictionary.

    Returns:
        dict: A dictionary containing the entities.
    """
    self.logger.info(f"Source type: {self.source_type}")
    if self.source_type == "python":
        return convert_enums(
            EntitiesDict(python_path=self.source_path).single_json()
        )
    elif self.source_type == "excel":
        return self.entities_to_json()
    else:
        raise NotImplementedError(f"Source type {self.source_type} not supported.")

bam_masterdata.parsing.parsing

AbstractParser

Bases: ABC

Example Abstract base class for parsers. Each parser should inherit from this class and implement the parse() method to populate collection.

Source code in bam_masterdata/parsing/parsing.py
class AbstractParser(ABC):
    """
    Example Abstract base class for parsers. Each parser should inherit from this class and implement
    the `parse()` method to populate `collection`.
    """

    @abstractmethod
    def parse(
        self,
        files: list[str],
        collection: CollectionType,
        logger: "BoundLoggerLazyProxy",
    ) -> None:
        """
        Parse the input `files` and populate the provided `collection` with object types, their metadata,
        and their relationships.

        Args:
            files (list[str]): List of file paths to be parsed.
            collection (CollectionType): Collection to be populated with parsed data.
            logger (BoundLoggerLazyProxy): Logger for logging messages during parsing.
        """
        pass

parse(files, collection, logger) abstractmethod

Parse the input files and populate the provided collection with object types, their metadata, and their relationships.

Parameters:

Name Type Description Default
files list[str]

List of file paths to be parsed.

required
collection CollectionType

Collection to be populated with parsed data.

required
logger BoundLoggerLazyProxy

Logger for logging messages during parsing.

required
Source code in bam_masterdata/parsing/parsing.py
@abstractmethod
def parse(
    self,
    files: list[str],
    collection: CollectionType,
    logger: "BoundLoggerLazyProxy",
) -> None:
    """
    Parse the input `files` and populate the provided `collection` with object types, their metadata,
    and their relationships.

    Args:
        files (list[str]): List of file paths to be parsed.
        collection (CollectionType): Collection to be populated with parsed data.
        logger (BoundLoggerLazyProxy): Logger for logging messages during parsing.
    """
    pass

bam_masterdata.utils.utils

code_to_class_name(code, logger=logger, entity_type='object')

Converts an openBIS code to a class name by capitalizing each word and removing special characters. In the special case the entity is a property type, it retains the full name separated by points instead of only keeping the last name (e.g., "TEM.INSTRUMENT" -> "TemInstrument" instead of "Instrument").

Parameters:

Name Type Description Default
code str

The openBIS code to convert to a class name.

required
logger BoundLoggerLazyProxy

The logger to log messages. Default is logger.

logger
entity_type str

The type of entity to convert. Default is "object".

'object'

Returns: str: The class name derived from the openBIS code.

Source code in bam_masterdata/utils/utils.py
def code_to_class_name(
    code: str | None,
    logger: "BoundLoggerLazyProxy" = logger,
    entity_type: str = "object",
) -> str:
    """
    Converts an openBIS `code` to a class name by capitalizing each word and removing special characters. In
    the special case the entity is a property type, it retains the full name separated by points instead of
    only keeping the last name (e.g., "TEM.INSTRUMENT" -> "TemInstrument" instead of "Instrument").

    Args:
        code (str): The openBIS code to convert to a class name.
        logger (BoundLoggerLazyProxy): The logger to log messages. Default is `logger`.
        entity_type (str): The type of entity to convert. Default is "object".
    Returns:
        str: The class name derived from the openBIS code.
    """
    if not code:
        logger.error(
            "The `code` is empty. Please, provide a proper input to the function."
        )
        return ""

    if entity_type == "property":
        code_names = chain.from_iterable(
            [c.split("_") for c in code.lstrip("$").split(".")]
        )
        return "".join(c.capitalize() for c in code_names)
    return "".join(c.capitalize() for c in code.lstrip("$").rsplit(".")[-1].split("_"))

delete_and_create_dir(directory_path, logger=logger, force_delete=False)

Deletes the directory at directory_path and creates a new one in the same path.

Parameters:

Name Type Description Default
directory_path str

The directory path to delete and create the folder.

required
logger BoundLoggerLazyProxy

The logger to log messages. Default is logger.

logger
force_delete bool

If True, the directory will be forcibly deleted if it exists.

False
Source code in bam_masterdata/utils/utils.py
def delete_and_create_dir(
    directory_path: str,
    logger: "BoundLoggerLazyProxy" = logger,
    force_delete: bool = False,
) -> None:
    """
    Deletes the directory at `directory_path` and creates a new one in the same path.

    Args:
        directory_path (str): The directory path to delete and create the folder.
        logger (BoundLoggerLazyProxy): The logger to log messages. Default is `logger`.
        force_delete (bool): If True, the directory will be forcibly deleted if it exists.
    """
    if not directory_path:
        logger.warning(
            "The `directory_path` is empty. Please, provide a proper input to the function."
        )
        return None

    if not force_delete:
        logger.info(f"Skipping the deletion of the directory at {directory_path}.")
        if not os.path.exists(directory_path):
            os.makedirs(directory_path)
        return None

    if os.path.exists(directory_path):
        try:
            shutil.rmtree(directory_path)  # ! careful with this line
        except PermissionError:
            logger.error(
                f"Permission denied to delete the directory at {directory_path}."
            )
            return None
    os.makedirs(directory_path)

duplicated_property_types(module_path, logger)

Find the duplicated property types in a module specified by module_path and returns a dictionary containing the duplicated property types class names as keys and the lines where they matched as values.

Parameters:

Name Type Description Default
module_path str

The path to the module containing the property types.

required
logger BoundLoggerLazyProxy

The logger to log messages.

required

Returns:

Name Type Description
dict dict

A dictionary containing the duplicated property types class names as keys and the

dict

lines where they matched as values.

Source code in bam_masterdata/utils/utils.py
def duplicated_property_types(module_path: str, logger: "BoundLoggerLazyProxy") -> dict:
    """
    Find the duplicated property types in a module specified by `module_path` and returns a dictionary
    containing the duplicated property types class names as keys and the lines where they matched as values.

    Args:
        module_path (str): The path to the module containing the property types.
        logger (BoundLoggerLazyProxy): The logger to log messages.

    Returns:
        dict: A dictionary containing the duplicated property types class names as keys and the
        lines where they matched as values.
    """
    duplicated_props: dict = {}
    module = import_module(module_path=module_path)
    source_code = inspect.getsource(module)
    for name, _ in inspect.getmembers(module):
        if name.startswith("_") or name == "PropertyTypeDef":
            continue

        pattern = rf"^\s*{name} *= *PropertyTypeDef"

        # Find all matching line numbers
        matches = [
            i + 1  # Convert to 1-based index
            for i, line in enumerate(source_code.splitlines())
            if re.match(pattern, line)
        ]
        if len(matches) > 1:
            duplicated_props[name] = matches
    if duplicated_props:
        logger.critical(
            f"Found {len(duplicated_props)} duplicated property types. These are stored in a dictionary "
            f"where the keys are the names of the variables in property_types.py and the values are the lines in the module: {duplicated_props}"
        )
    return duplicated_props

format_json_id(value)

Converts snake_case or UPPER_CASE to PascalCase while keeping special cases like '$NAME' untouched.

Source code in bam_masterdata/utils/utils.py
def format_json_id(value):
    """Converts snake_case or UPPER_CASE to PascalCase while keeping special cases like '$NAME' untouched."""
    if value.startswith("$"):
        # Remove "$" and apply PascalCase transformation
        value = value[1:]
    return "".join(
        word.capitalize() for word in re.split(r"[\._]", value)
    )  # PascalCase

import_module(module_path)

Dynamically imports a module from the given file path.

Parameters:

Name Type Description Default
module_path str

Path to the Python module file.

required

Returns:

Name Type Description
module Any

Imported module object.

Source code in bam_masterdata/utils/utils.py
def import_module(module_path: str) -> Any:
    """
    Dynamically imports a module from the given file path.

    Args:
        module_path (str): Path to the Python module file.

    Returns:
        module: Imported module object.
    """
    module_name = os.path.splitext(os.path.basename(module_path))[0]
    spec = importlib.util.spec_from_file_location(module_name, module_path)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module

is_reduced_version(generated_code_value, code)

Check if generated_code_value is a reduced version of code.

Parameters:

Name Type Description Default
generated_code_value str

The potentially reduced code.

required
code str

The original full code.

required

Returns:

Name Type Description
bool bool

True if generated_code_value is a reduced version of code, False otherwise.

Source code in bam_masterdata/utils/utils.py
def is_reduced_version(generated_code_value: str, code: str) -> bool:
    """
    Check if generated_code_value is a reduced version of code.

    Args:
        generated_code_value (str): The potentially reduced code.
        code (str): The original full code.

    Returns:
        bool: True if generated_code_value is a reduced version of code, False otherwise.
    """
    if generated_code_value == "" or code == "":
        return False

    if code.startswith(generated_code_value):
        return True

    # Check if both are single words (no delimiters)
    if not any(delimiter in code for delimiter in "._") and not any(
        delimiter in generated_code_value for delimiter in "._"
    ):
        return True

    # Determine the delimiter in each string
    code_delimiter = "." if "." in code else "_" if "_" in code else None
    generated_delimiter = (
        "."
        if "." in generated_code_value
        else "_"
        if "_" in generated_code_value
        else None
    )

    # If delimiters don't match, return False
    if code_delimiter != generated_delimiter:
        return False

    # Split both strings using the determined delimiter
    generated_parts = generated_code_value.split(code_delimiter)
    original_parts = code.split(code_delimiter)

    # Ensure both have the same number of parts
    return len(generated_parts) == len(original_parts)

listdir_py_modules(directory_path, logger=logger)

Recursively goes through the directory_path and returns a list of all .py files that do not start with '_'. If directory_path is a single Python module file, it will return a list with that file.

Parameters:

Name Type Description Default
directory_path str

The directory path to search through.

required
logger BoundLoggerLazyProxy

The logger to log messages. Default is logger.

logger

Returns:

Type Description
list[str]

list[str]: A list of all .py files that do not start with '_'

Source code in bam_masterdata/utils/utils.py
def listdir_py_modules(
    directory_path: str, logger: "BoundLoggerLazyProxy" = logger
) -> list[str]:
    """
    Recursively goes through the `directory_path` and returns a list of all .py files that do not start with '_'. If
    `directory_path` is a single Python module file, it will return a list with that file.

    Args:
        directory_path (str): The directory path to search through.
        logger (BoundLoggerLazyProxy): The logger to log messages. Default is `logger`.

    Returns:
        list[str]: A list of all .py files that do not start with '_'
    """
    if not directory_path:
        logger.warning(
            "The `directory_path` is empty. Please, provide a proper input to the function."
        )
        return []

    # In case of a individual Python module file
    if directory_path.endswith(".py"):
        return [directory_path]
    # Use glob to find all .py files recursively in a directory containing all modules
    else:
        files = glob.glob(os.path.join(directory_path, "**", "*.py"), recursive=True)
    if not files:
        logger.info("No Python files found in the directory.")
        return []

    # Filter out files that start with '_'
    # ! sorted in order to avoid using with OS sorting differently
    return sorted(
        [
            f
            for f in files
            if not os.path.basename(f).startswith("_") and "tmp" not in f.split(os.sep)
        ]
    )

store_log_message(logger, entity_ref, message, level='error')

Logs a message and stores it inside the entity's _log_msgs list.

Parameters:

Name Type Description Default
entity_ref dict

The entity dictionary where _log_msgs should be stored.

required
message str

The log message.

required
level str

Log level ('error', 'warning', 'critical', 'info').

'error'
Source code in bam_masterdata/utils/utils.py
def store_log_message(logger, entity_ref, message, level="error"):
    """
    Logs a message and stores it inside the entity's _log_msgs list.

    Args:
        entity_ref (dict): The entity dictionary where _log_msgs should be stored.
        message (str): The log message.
        level (str): Log level ('error', 'warning', 'critical', 'info').
    """
    log_function = {
        "error": logger.error,
        "warning": logger.warning,
        "critical": logger.critical,
        "info": logger.info,
    }.get(level, logger.error)

    # Log the message
    log_function(message)

    # Ensure _log_msgs exists
    if "_log_msgs" not in entity_ref:
        entity_ref["_log_msgs"] = []

    # Append log message
    entity_ref["_log_msgs"].append((level, message))

bam_masterdata.utils.paths

find_dir(possible_locations)

Search for a valid directory in a list of possible locations.

Parameters:

Name Type Description Default
possible_locations list[Path]

A list of possible locations to search for a directory.

required

Raises:

Type Description
FileNotFoundError

If no valid directory is found.

Returns:

Name Type Description
str str

The path of the valid directory.

Source code in bam_masterdata/utils/paths.py
def find_dir(possible_locations: list[Path]) -> str:
    """
    Search for a valid directory in a list of possible locations.

    Args:
        possible_locations (list[Path]): A list of possible locations to search for a directory.

    Raises:
        FileNotFoundError: If no valid directory is found.

    Returns:
        str: The path of the valid directory.
    """
    for path in possible_locations:
        if path.exists():
            return str(path.resolve())

    raise FileNotFoundError("Could not find a valid directory.")

bam_masterdata.utils.users

UserID

Source code in bam_masterdata/utils/users.py
class UserID:
    def __init__(self, url: str = ""):
        if not url:
            raise ValueError("Missing url to connect to openBIS")
        self.openbis = ologin(url=url)
        self.users = self.openbis.get_users()

    def _split_name(self, name: str):
        """
        Split a full name into firstname and lastname using comma ',' or space ' ' as separator.

        Args:
            name (str): Full name to split, e.g., "John Doe" or "Doe, John"
        """
        parts = re.split(r",|\s+", name.strip())
        parts = [p for p in parts if p]
        if len(parts) >= 2:
            return parts[0], parts[1]
        return parts[0], ""  # if only one name

    def get_userid_from_names(self, firstname: str, lastname: str) -> str | None:
        """
        Return the userId matching the given first and last name (case-insensitive).

        Args:
            firstname (str): First name.
            lastname (str): Last name.

        Returns:
            str | None: The userId if a match is found, otherwise None.
        """
        for u in self.users:
            if (
                u.firstName.lower() == firstname.lower()
                and u.lastName.lower() == lastname.lower()
            ):
                return u.userId
        return None

    def get_userid_from_fullname(self, name: str) -> str | None:
        """
        Return the userId matching the given fullname (case-insensitive). It uses the `_split_name` function.

        Args:
            name (str): Full name, e.g., "John Doe" or "Doe, John".

        Returns:
            str | None: The userId if a match is found, otherwise None.
        """
        firstname, lastname = self._split_name(name)
        for u in self.users:
            if (u.firstName.lower(), u.lastName.lower()) == (
                firstname.lower(),
                lastname.lower(),
            ) or (u.firstName.lower(), u.lastName.lower()) == (
                lastname.lower(),
                firstname.lower(),
            ):
                return u.userId
        return None

get_userid_from_fullname(name)

Return the userId matching the given fullname (case-insensitive). It uses the _split_name function.

Parameters:

Name Type Description Default
name str

Full name, e.g., "John Doe" or "Doe, John".

required

Returns:

Type Description
str | None

str | None: The userId if a match is found, otherwise None.

Source code in bam_masterdata/utils/users.py
def get_userid_from_fullname(self, name: str) -> str | None:
    """
    Return the userId matching the given fullname (case-insensitive). It uses the `_split_name` function.

    Args:
        name (str): Full name, e.g., "John Doe" or "Doe, John".

    Returns:
        str | None: The userId if a match is found, otherwise None.
    """
    firstname, lastname = self._split_name(name)
    for u in self.users:
        if (u.firstName.lower(), u.lastName.lower()) == (
            firstname.lower(),
            lastname.lower(),
        ) or (u.firstName.lower(), u.lastName.lower()) == (
            lastname.lower(),
            firstname.lower(),
        ):
            return u.userId
    return None

get_userid_from_names(firstname, lastname)

Return the userId matching the given first and last name (case-insensitive).

Parameters:

Name Type Description Default
firstname str

First name.

required
lastname str

Last name.

required

Returns:

Type Description
str | None

str | None: The userId if a match is found, otherwise None.

Source code in bam_masterdata/utils/users.py
def get_userid_from_names(self, firstname: str, lastname: str) -> str | None:
    """
    Return the userId matching the given first and last name (case-insensitive).

    Args:
        firstname (str): First name.
        lastname (str): Last name.

    Returns:
        str | None: The userId if a match is found, otherwise None.
    """
    for u in self.users:
        if (
            u.firstName.lower() == firstname.lower()
            and u.lastName.lower() == lastname.lower()
        ):
            return u.userId
    return None

get_bam_username(firstname, lastname)

Tries to get the BAM username from the first and last names. The BAM username guess is defined by concatenating the first letter of the first name with the, max. 7 digits of the last name.

Parameters:

Name Type Description Default
firstname str

The first name.

required
lastname str

The last name.

required

Returns:

Name Type Description
str str

The guessed BAM username.

Source code in bam_masterdata/utils/users.py
def get_bam_username(firstname: str, lastname: str) -> str:
    """
    Tries to get the BAM username from the first and last names. The BAM username guess is defined by concatenating
    the first letter of the first name with the, max. 7 digits of the last name.

    Args:
        firstname (str): The first name.
        lastname (str): The last name.

    Returns:
        str: The guessed BAM username.
    """

    def _de_replacements(name: str) -> str:
        """
        Perform common German name replacements to standardize the name.

        Args:
            name (str): Name to standardize.

        Returns:
            str: Standardized name.
        """
        replacements = {
            "Ä": "Ae",
            "Ö": "Oe",
            "Ü": "Ue",
            "ä": "ae",
            "ö": "oe",
            "ü": "ue",
            "ß": "ss",
        }
        for old, new in replacements.items():
            name = name.replace(old, new)
        return name

    # German umlaut replacements
    firstname = _de_replacements(firstname).lower().strip()
    lastname = _de_replacements(lastname).lower().strip()

    # Defining username format
    first_letter = firstname[0]
    if len(lastname) > 7:
        last_part = lastname[:7]
    else:
        last_part = lastname
    return f"{first_letter}{last_part}".upper()