From 387d60eff22121947550139f3e8c79bfaec470be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Clau=C3=9F?= Date: Wed, 24 May 2023 13:17:00 +0200 Subject: [PATCH] helper recursice data fetch --- src/paveit/datamodels/material.py | 31 +++++-------- src/paveit/datamodels/norm_specification.py | 25 ++++++----- src/paveit/helper/__init__.py | 4 +- src/paveit/helper/mongo.py | 50 ++++++++++++++++++++- 4 files changed, 76 insertions(+), 34 deletions(-) diff --git a/src/paveit/datamodels/material.py b/src/paveit/datamodels/material.py index 268c349..31ad356 100755 --- a/src/paveit/datamodels/material.py +++ b/src/paveit/datamodels/material.py @@ -4,10 +4,15 @@ from re import T from bson.json_util import loads from mongoengine import * -from .enumeration import BitumenCategoryEnum -from .norm_specification import DeliveryGrain, EnumerateBase -from .norm_documents import NormDocumentAggregate, NormDocumentBitumen, NormDocumentAsphalt +from paveit.helper import fetch_recursive, mongo_to_dict +from .enumeration import BitumenCategoryEnum +from .norm_documents import ( + NormDocumentAggregate, + NormDocumentAsphalt, + NormDocumentBitumen, +) +from .norm_specification import DeliveryGrain, EnumerateBase from .project import Project from .usermanagement import Organisation, User @@ -36,25 +41,11 @@ class Material(Document): def to_dict(self): # convert data to dict - data = self.to_mongo().to_dict() - - # Laden und hinzufügen der referenzierten Daten - try: - data['parameter'] = self.parameter.to_mongo().to_dict() - except: - pass - - try: - norm = self.norm.fetch() # Trigger the dereferencing - data['norm'] = norm.to_mongo().to_dict() - except: - raise - + data = fetch_recursive(self) + data = mongo_to_dict(data) + return data - - - meta = { 'allow_inheritance': True, 'index_opts': {}, diff --git a/src/paveit/datamodels/norm_specification.py b/src/paveit/datamodels/norm_specification.py index 0dea4bf..e538f27 100644 --- a/src/paveit/datamodels/norm_specification.py +++ b/src/paveit/datamodels/norm_specification.py @@ -2,8 +2,18 @@ from mongoengine import * from .enumeration import RelationalOperatorsEnum, BitumenCategoryEnum from .norm_documents import NormDocumentAggregate, NormDocumentBitumen, NormDocumentAsphalt + +from paveit.helper import fetch_recursive, mongo_to_dict + class EnumerateBase(Document): + def to_dict(self): + # convert data to dict + data = fetch_recursive(self) + data = mongo_to_dict(data) + print(data) + return data + meta = { 'allow_inheritance': True, 'index_opts': {}, @@ -13,6 +23,8 @@ class EnumerateBase(Document): 'collection': 'norm_specification', "db_alias": 'dblabtests', } + + # Gesteine class DeliveryGrain(EnumerateBase): @@ -21,10 +33,10 @@ class DeliveryGrain(EnumerateBase): norm = LazyReferenceField(NormDocumentAggregate, required=True) # Bitumen +class BitumenParameterLimitsBaseEnum(EnumerateBase): + pass - - -class BitumenParameterLimitsStrassenbaubitumen(EnumerateBase): +class BitumenParameterLimitsStrassenbaubitumen(BitumenParameterLimitsBaseEnum): name = StringField() category = EnumField(BitumenCategoryEnum, required=True) @@ -73,10 +85,3 @@ class BitumenParameterLimitsStrassenbaubitumen(EnumerateBase): hardening_resistance_masschange_operator = EnumField(RelationalOperatorsEnum, default=RelationalOperatorsEnum.lt) hardening_resistance_masschange_max = FloatField(min_value=0, max_value=100) - - -# Asphalt -class Asphalt(EnumerateBase): - name = StringField() - category = StringField() - norm = LazyReferenceField(NormDocumentAsphalt, required=True) diff --git a/src/paveit/helper/__init__.py b/src/paveit/helper/__init__.py index 911540d..17e418d 100755 --- a/src/paveit/helper/__init__.py +++ b/src/paveit/helper/__init__.py @@ -1,10 +1,10 @@ from .filehandling import read_file_to_bytesio from .filehasher import calc_hash_of_bytes from .minio import get_minio_client_archive, get_minio_client_processing -from .mongo import connect_mongo_db, mongo_get_results +from .mongo import connect_mongo_db, fetch_recursive, mongo_get_results, mongo_to_dict __all__ = [ - 'read_file_to_bytesio', 'connect_mongo_db', 'mongo_get_results', + 'read_file_to_bytesio', 'connect_mongo_db', 'mongo_get_results', 'fetch_recursive', 'mongo_to_dict', 'get_minio_client_processing', 'get_minio_client_archive', 'calc_hash_of_bytes' ] diff --git a/src/paveit/helper/mongo.py b/src/paveit/helper/mongo.py index f50e0fe..409ba4a 100755 --- a/src/paveit/helper/mongo.py +++ b/src/paveit/helper/mongo.py @@ -1,7 +1,10 @@ +import os + +import mongoengine from bson import ObjectId from mongoengine import connect as mongo_connect from pandas import DataFrame -import os + def connect_mongo_db(username=os.environ['MONGO_USER'] , password=os.environ['MONGO_PASSWD'] , @@ -65,4 +68,47 @@ def mongo_get_results(resultsmodel, results: DataFrame, datamodel, project_id: ObjectId, material_id: ObjectId, user_id: ObjectId): - return True \ No newline at end of file + return True + +def fetch_recursive(data, fetch_parameter=['norm']): + + fields = data._fields + + data_out = data.to_mongo().to_dict() + + for par in fetch_parameter: + if par in fields.keys(): + + try: + # if is LazyReferenceField + if isinstance(fields[par], mongoengine.fields.LazyReferenceField): + d = data[par].fetch() + else: + d = data[par] + + except: + continue + + if d is None: + continue + + data_out[par] = d.to_mongo().to_dict() + + return data_out + +def mongo_to_dict(data, drop_parameters=['_cls','user_id', 'org_id', 'project_id']): + ''' + data: dict + + ''' + for key in list(data.keys()): + if key in drop_parameters: + del data[key] # Remove the unwanted key + elif isinstance(data[key], dict): + mongo_to_dict(data[key]) # Recurse into nested dictionaries + else: + # process data + if isinstance(data[key], ObjectId): + data[key] = str(data[key]) + + return data \ No newline at end of file