From a27c94eb7c6c693815da187ab39fa83661023be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Clau=C3=9F?= Date: Fri, 2 Jun 2023 08:53:44 +0200 Subject: [PATCH] add more requests --- src/paveit/datamodels/citt.py | 46 ++++----- src/paveit/datamodels/data.py | 4 +- src/paveit/datamodels/enumeration.py | 7 +- src/paveit/datamodels/sheartest.py | 46 ++++----- src/paveit/datamodels/taskmanager.py | 8 +- src/paveit/helper/minio.py | 4 +- src/paveit/helper/mongo.py | 2 +- src/paveit/labtest/base.py | 51 ++++++--- src/paveit/labtest/citt.py | 149 +++++++++++++++++++++------ src/paveit/labtest/sheartest.py | 12 ++- 10 files changed, 220 insertions(+), 109 deletions(-) diff --git a/src/paveit/datamodels/citt.py b/src/paveit/datamodels/citt.py index 48442a0..6991800 100755 --- a/src/paveit/datamodels/citt.py +++ b/src/paveit/datamodels/citt.py @@ -34,7 +34,7 @@ class CyclicIndirectTensileTest(Document): 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'collection': 'citt', + 'collection': 'lab_citt', "db_alias": 'dblabtests', } @@ -152,25 +152,25 @@ class CITTSiffnessResults(CyclicIndirectTensileTest): #optional parameter ## Piston - s_hor_piston_amp = FloatField(required=False) - s_hor_piston_freq = FloatField(required=False) - s_hor_piston_phase = FloatField(required=False) - s_hor_piston_offset = FloatField(required=False) - s_hor_piston_slope = FloatField(required=False) - s_hor_piston_r2 = FloatField(required=False) - s_hor_piston_cycle_min = ListField(FloatField(),required=False) - s_hor_piston_min = FloatField(required=False) - s_hor_piston_min_std = FloatField(required=False) - s_hor_piston_min_diff_rel = FloatField(required=False) - s_hor_piston_cycle_max = ListField(FloatField(),required=False) - s_hor_piston_max = FloatField(required=False) - s_hor_piston_max_std = FloatField(required=False) - s_hor_piston_max_dif_rel = FloatField(required=False) - s_hor_piston_cycle_mean = ListField(FloatField(),required=False) - s_hor_piston_mean = FloatField(required=False) - s_hor_piston_mean_std = FloatField(required=False) - s_hor_piston_mean_diff_rel = FloatField(required=False) - s_hor_piston_cycle_diff = ListField(FloatField(),required=False) - s_hor_piston_diff = FloatField(required=False) - s_hor_piston_diff_std = FloatField(required=False) - s_hor_piston_diff_diff_rel = FloatField(required=False) \ No newline at end of file + s_piston_amp = FloatField(required=False) + s_piston_freq = FloatField(required=False) + s_piston_phase = FloatField(required=False) + s_piston_offset = FloatField(required=False) + s_piston_slope = FloatField(required=False) + s_piston_r2 = FloatField(required=False) + s_piston_cycle_min = ListField(FloatField(),required=False) + s_piston_min = FloatField(required=False) + s_piston_min_std = FloatField(required=False) + s_piston_min_diff_rel = FloatField(required=False) + s_piston_cycle_max = ListField(FloatField(),required=False) + s_piston_max = FloatField(required=False) + s_piston_max_std = FloatField(required=False) + s_piston_max_diff_rel = FloatField(required=False) + s_piston_cycle_mean = ListField(FloatField(),required=False) + s_piston_mean = FloatField(required=False) + s_piston_mean_std = FloatField(required=False) + s_piston_mean_diff_rel = FloatField(required=False) + s_piston_cycle_diff = ListField(FloatField(),required=False) + s_piston_diff = FloatField(required=False) + s_piston_diff_std = FloatField(required=False) + s_piston_diff_diff_rel = FloatField(required=False) \ No newline at end of file diff --git a/src/paveit/datamodels/data.py b/src/paveit/datamodels/data.py index 3a65cb2..89bf983 100755 --- a/src/paveit/datamodels/data.py +++ b/src/paveit/datamodels/data.py @@ -23,7 +23,7 @@ class RawSinData(Document): 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'collection': 'RawSinData', + 'collection': 'data_rawsine', "db_alias": 'dblabtests', } @@ -40,7 +40,7 @@ class RawData(Document): 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'collection': 'RawSinData', + 'collection': 'data_raw', "db_alias": 'dblabtests', } diff --git a/src/paveit/datamodels/enumeration.py b/src/paveit/datamodels/enumeration.py index dd46ebf..de0b7d5 100755 --- a/src/paveit/datamodels/enumeration.py +++ b/src/paveit/datamodels/enumeration.py @@ -59,10 +59,13 @@ class Config(Document): # convert data to dict data = fetch_recursive(self) data = mongo_to_dict(data) + print(data) if enabled: - data = list(filter(lambda item: item["enabled"], data)) - + if data['enabled']: + return data + else: + return [] return data meta = { diff --git a/src/paveit/datamodels/sheartest.py b/src/paveit/datamodels/sheartest.py index 6e90801..f7c7a74 100755 --- a/src/paveit/datamodels/sheartest.py +++ b/src/paveit/datamodels/sheartest.py @@ -38,7 +38,7 @@ class DynamicShearTest(Document): "db_alias": 'dblabtests', 'collection': - 'sheartest', + 'lab_sheartest', 'indexes': [ [("lab", 1)], [("speciment_name", 1)], @@ -213,25 +213,25 @@ class DynamicShearTestExtension(DynamicShearTest): s_hor_2_diff_std = FloatField(required=False) s_hor_2_diff_diff_rel = FloatField(required=False) ## Piston - s_hor_piston_amp = FloatField(required=False) - s_hor_piston_freq = FloatField(required=False) - s_hor_piston_phase = FloatField(required=False) - s_hor_piston_offset = FloatField(required=False) - s_hor_piston_slope = FloatField(required=False) - s_hor_piston_r2 = FloatField(required=False) - s_hor_piston_cycle_min = ListField(FloatField(),required=False) - s_hor_piston_min = FloatField(required=False) - s_hor_piston_min_std = FloatField(required=False) - s_hor_piston_min_diff_rel = FloatField(required=False) - s_hor_piston_cycle_max = ListField(FloatField(),required=False) - s_hor_piston_max = FloatField(required=False) - s_hor_piston_max_std = FloatField(required=False) - s_hor_piston_max_dif_rel = FloatField(required=False) - s_hor_piston_cycle_mean = ListField(FloatField(),required=False) - s_hor_piston_mean = FloatField(required=False) - s_hor_piston_mean_std = FloatField(required=False) - s_hor_piston_mean_diff_rel = FloatField(required=False) - s_hor_piston_cycle_diff = ListField(FloatField(),required=False) - s_hor_piston_diff = FloatField(required=False) - s_hor_piston_diff_std = FloatField(required=False) - s_hor_piston_diff_diff_rel = FloatField(required=False) \ No newline at end of file + s_piston_amp = FloatField(required=False) + s_piston_freq = FloatField(required=False) + s_piston_phase = FloatField(required=False) + s_piston_offset = FloatField(required=False) + s_piston_slope = FloatField(required=False) + s_piston_r2 = FloatField(required=False) + s_piston_cycle_min = ListField(FloatField(),required=False) + s_piston_min = FloatField(required=False) + s_piston_min_std = FloatField(required=False) + s_piston_min_diff_rel = FloatField(required=False) + s_piston_cycle_max = ListField(FloatField(),required=False) + s_piston_max = FloatField(required=False) + s_piston_max_std = FloatField(required=False) + s_piston_max_dif_rel = FloatField(required=False) + s_piston_cycle_mean = ListField(FloatField(),required=False) + s_piston_mean = FloatField(required=False) + s_piston_mean_std = FloatField(required=False) + s_piston_mean_diff_rel = FloatField(required=False) + s_piston_cycle_diff = ListField(FloatField(),required=False) + s_piston_diff = FloatField(required=False) + s_piston_diff_std = FloatField(required=False) + s_piston_diff_diff_rel = FloatField(required=False) \ No newline at end of file diff --git a/src/paveit/datamodels/taskmanager.py b/src/paveit/datamodels/taskmanager.py index 99e67e0..808599f 100755 --- a/src/paveit/datamodels/taskmanager.py +++ b/src/paveit/datamodels/taskmanager.py @@ -3,18 +3,20 @@ import datetime from mongoengine import * -from .infrastructure import MachineBase +from .machines import MachineBase from .material import Material from .project import Project from .usermanagement import Organisation, User from .workpackage import Workpackage from .enumeration import ProcessStatusEnum +from .client import Client class TaskManagerBase(Document): org_id = LazyReferenceField(Organisation, required=True) user_id = LazyReferenceField(User, required=True) + client_id = LazyReferenceField(Client, required=True) project_id = LazyReferenceField(Project, required=True) wp_id = LazyReferenceField(Workpackage, required=False) @@ -28,7 +30,7 @@ class TaskManagerBase(Document): task_finished = DateTimeField(required=False) assign_users = ListField(LazyReferenceField(User), required=False) - assign_machines = ListField(LazyReferenceField(MachineBase), required=False) + assign_machine = LazyReferenceField(MachineBase, required=False) series = StringField(default='Serie 01') @@ -50,6 +52,6 @@ class TaskCITTStiffness(TaskManagerBase): class TaskDynShearStiffness(TaskManagerBase): - material1 = LazyReferenceField(Material, required=True) + material = LazyReferenceField(Material, required=True) material2 = LazyReferenceField(Material, required=True) bounding = LazyReferenceField(Material, required=True) \ No newline at end of file diff --git a/src/paveit/helper/minio.py b/src/paveit/helper/minio.py index e9410cb..e10ba42 100755 --- a/src/paveit/helper/minio.py +++ b/src/paveit/helper/minio.py @@ -9,7 +9,7 @@ def get_minio_client_processing(bucket_name = 'processing'): os.environ["MINIO_URL"], access_key=os.environ["MINIO_ACCESS_KEY"], secret_key=os.environ["MINIO_SECRET_KEY"], - secure=False + secure=True ) @@ -28,7 +28,7 @@ def get_minio_client_archive(bucket_name = 'archive'): os.environ["MINIO_ARCHIVE_URL"], access_key=os.environ["MINIO_ARCHIVE_ACCESS_KEY"], secret_key=os.environ["MINIO_ARCHIVE_SECRET_KEY"], - secure=False + secure=True ) found = client.bucket_exists(bucket_name) diff --git a/src/paveit/helper/mongo.py b/src/paveit/helper/mongo.py index 6f461b3..7605725 100755 --- a/src/paveit/helper/mongo.py +++ b/src/paveit/helper/mongo.py @@ -70,7 +70,7 @@ def mongo_get_results(resultsmodel, results: DataFrame, datamodel, return True -def fetch_recursive(data, fetch_parameter=['norm', 'limits']): +def fetch_recursive(data, fetch_parameter=['norm', 'limits', 'assign_machine']): fields = data._fields diff --git a/src/paveit/labtest/base.py b/src/paveit/labtest/base.py index d7b905d..c1f65ef 100755 --- a/src/paveit/labtest/base.py +++ b/src/paveit/labtest/base.py @@ -40,12 +40,6 @@ class DataSineLoad(): self._pre_run() - def _which_machine(self): - """ - check the file and try to get the machine from the data - """ - pass - def _set_parameter(self): self._logger.debug('run _set_parameter') @@ -57,7 +51,7 @@ class DataSineLoad(): ] self.val_col_names = [ - 'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2' + 'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2', 's_piston' ] self.columns_analyse = [ @@ -110,7 +104,7 @@ class DataSineLoad(): response.release_conn() self.data = io.BytesIO(self.data) - self._logger.debug('data read from s3') + self._logger.debug('read data from s3') def _calc_hash_of_bytesio(self): self._logger.debug('run _calc_hash_of_bytesio') @@ -145,6 +139,11 @@ class DataSineLoad(): for key, d in self.metadata.items(): try: + #remove units + for unit in ["°C", 'Hz']: + if unit in d: + d = d.split(unit)[0].strip() + f = float(d.replace(',', '.')) self.metadata[key] = f except: @@ -160,9 +159,27 @@ class DataSineLoad(): colnames = [sub.replace(name, par) for sub in colnames] self.data.columns = colnames + + self._logger.debug(f'columns: {colnames}') + + print(self.data.head()) + print(self.data['s_piston'].head()) + + self._logger.debug(f'standardize_data: {self.data.columns}') + def _standardize_meta(self): self._logger.debug('run _standardize_meta') + + # remove "\r\n" ending from Windows + for col in list(self.metadata.keys()): + + col_mod = col.replace('\r\n', '') + + if col != col_mod: + self.metadata[col_mod] = self.metadata[col] + self.metadata.pop(col) + for par, names in self.meta_names_of_parameter.items(): for name in names: @@ -172,6 +189,8 @@ class DataSineLoad(): self.metadata.pop(name) break + + self._logger.debug(f'meta (stand.): {self.metadata}') def _modify_meta(self): pass @@ -184,13 +203,14 @@ class DataSineLoad(): # check if value in metadata: if name in self.metadata.keys(): - + self._logger.error(f'add {name} from metadata to data') self.data[name] = self.metadata[name] else: - - print(name) + self._logger.error(f'{name} not in data') raise + + self._logger.debug(f'validate_data: {self.data.columns}') def _validate_meta(self): self._logger.debug('run _validate_meta') @@ -295,7 +315,7 @@ class DataSineLoad(): gaps = idx_diff > (4 * dt_mean) has_gaps = any(gaps) - + if has_gaps == False: data_list.append(d) @@ -359,8 +379,7 @@ class DataSineLoad(): self._calc_hash_of_bytesio() self._define_data_models() - self._data_in_db() - self._which_machine() + #self._data_in_db() self._set_parameter() self.update_parameter() self._define_units() @@ -371,11 +390,11 @@ class DataSineLoad(): self._process_data() self._meta_to_float() - self._standardize_data() self._standardize_meta() + self._standardize_data() self._modify_meta() - self._validate_data() self._validate_meta() + self._validate_data() self._post_string_to_float() self._post_select_importent_columns() diff --git a/src/paveit/labtest/citt.py b/src/paveit/labtest/citt.py index 6293bd1..202d9ba 100755 --- a/src/paveit/labtest/citt.py +++ b/src/paveit/labtest/citt.py @@ -13,6 +13,45 @@ from paveit.labtest import DataSineLoad class CITTBase(DataSineLoad): + + def _set_parameter(self): + self._logger.debug('run _set_parameter') + + self.split_data_based_on_parameter = ['T', 'sigma', 'f'] + + self.col_as_int = ['N'] + + self.col_as_float = [ + 'T', 'F', 's_piston', 's_hor_1', 'f', 's_hor_1', 's_hor_2' + ] + + self.val_col_names = [ + 'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2', 's_piston' + ] + + self.columns_analyse = [ + 'F', 's_hor_sum', 's_hor_1', 's_hor_2', 's_piston' + ] + + self.round_values = [('T', 3), ('sigma', 3)] + + # Header names after standardization; check if exists + self.val_header_names = ['speciment_height', 'speciment_diameter'] + + self.number_of_load_cycles_for_analysis = 5 + + #Dummy Data, replace in Machine Config + self.meta_names_of_parameter = { + 'sigma': ['Max. Spannung'] + } #list of names + + #Dummy Data, replace in Machine Config + self.data_column_names = { + 'time': ['Time Series'], + 'F': ['Load Series'], + 's_hor_1': ['LVDT1 Series'], + 's_hor_2': ['LVDT2 Series'], + } def _sel_df(self, df, num=5, shift=-1): @@ -117,8 +156,10 @@ class CITTBase(DataSineLoad): if data is None: continue if len(data) < 10: continue - + try: + + self._logger.debug(f'run fit on subset {idx_data}') data.index = data.index - data.index[0] @@ -200,8 +241,11 @@ class CITTBase(DataSineLoad): # TODO: Überarbeiten und erweitern (ISSUE #2) res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[ 'fit_s_hor_sum_phase'] - except: + except Exception as e: + self._logger.exception(e) res_temp = None + + self._logger.debug(res_temp) self.fit.append(res_temp) @@ -266,54 +310,90 @@ class CITTBase(DataSineLoad): s_hor_2=list(data['s_hor_2']), s_hor_sum=list(data['s_hor_sum']), ) + + self._logger.debug(f'columns data, {data.columns}') # add optional datas for col in ['s_piston']: if col in data.columns: + self._logger.debug(f'add {col} to output data') data_out[col] = list(data[col]) + outkeys = list(data_out.keys()) + self._logger.debug(f'write raw data to db, {outkeys}') g = CITTSiffness(result=f.id, **data_out).save() - -class CITT_TUDresden(CITTBase): - - def _which_machine(self): - """ - check the file and try to get the machine from the data - """ - - self._machine = 'TIRA' +class CITT_TUDresdenWille(CITTBase): def _define_units(self): - if self._machine == 'TIRA': - self.unit_s = 1 #mm - self.unit_F = 1. #N - self.unit_t = 1. #s + self.unit_s = 1 / 1000. #mm + self.unit_F = 1.0 #N + self.unit_t = 1. #s def update_parameter(self): - if self._machine == 'TIRA': + self.meta_names_of_parameter = { + 'sigma': ['Oberspannung'], + 'f': ['Prüffrequenz'], + 'T': ['Prüftemperatur'], + 't': ['Zeit'], + 'speciment_diameter': ['Probekörberdurchmesser', 'Probekörberbreite'], + 'speciment_height': ['Probekörperhöhe'], + } #list of names - self.meta_names_of_parameter = { - 'sigma': ['Oberspannung'], - 'f': ['Sollfrequnz'], - 'T': ['Solltemperatur'], - #'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'], - #'Nto': ['Letzer Aufzeichnungslastwechsel', 'Last Cycle'], - 't': ['Zeit'], - 'speciment_diameter': ['PK-Durchmesser'], - 'speciment_height': ['PK-Höhe'], - } #list of names + self.data_column_names = { + 'time': ['Zeit'], + 'F': ['Kraft'], + 'N': ['Zyklenzähler'], + 's_hor_1': ['Horizontalweg vorn', 'Weg Probe'], + 's_hor_2': ['Horizontalweg hinten', 'Weg 2'], + 's_piston': ['Kolbenweg'], + } - self.data_column_names = { - 'time': ['Zeit'], - 'F': ['Kraft'], - 'N': ['Lastwechsel'], - 's_hor_1': ['IWA_1 2 mm'], - 's_hor_2': ['IWA_2 2 mm'], - 's_piston': ['Kolbenweg'], - } + def _process_data(self): + + meta, data = read_geosys(self.data, '039', metadata_ids=['001','003', '011', '019']) + + #define in class + self.data = data.reset_index() + self.metadata.update(meta) + + # log infos + self._logger.debug(f'metadata: {self.metadata}') + self._logger.debug(f'data: {self.data.head()}') + + print(data.head()) + +class CITT_TUDresdenTira(CITTBase): + + def _define_units(self): + + self.unit_s = 1 #mm + self.unit_F = 1. #N + self.unit_t = 1. #s + + def update_parameter(self): + + self.meta_names_of_parameter = { + 'sigma': ['Oberspannung'], + 'f': ['Sollfrequnz'], + 'T': ['Solltemperatur'], + #'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'], + #'Nto': ['Letzer Aufzeichnungslastwechsel', 'Last Cycle'], + 't': ['Zeit'], + 'speciment_diameter': ['PK-Durchmesser', 'Probekörberbreite'], + 'speciment_height': ['PK-Höhe', 'Probekörperhöhe'], + } #list of names + + self.data_column_names = { + 'time': ['Zeit'], + 'F': ['Kraft'], + 'N': ['Lastwechsel'], + 's_hor_1': ['IWA_1 2 mm'], + 's_hor_2': ['IWA_2 2 mm'], + 's_piston': ['Kolbenweg'], + } def _process_data(self): @@ -396,6 +476,7 @@ class CITT_TUDresden(CITTBase): self._logger.info(self.data.head()) + class CITT_PTMDortmund(CITTBase): def _define_units(self): diff --git a/src/paveit/labtest/sheartest.py b/src/paveit/labtest/sheartest.py index e2d4459..d4f83ae 100755 --- a/src/paveit/labtest/sheartest.py +++ b/src/paveit/labtest/sheartest.py @@ -233,6 +233,8 @@ class ShearTestExtension(ShearTest): self._logger.info('run _calc') self.fit = [] + + errors = 0 for idx_data, data in enumerate(self.data): @@ -326,11 +328,13 @@ class ShearTestExtension(ShearTest): self.fit.append(res_temp) - if (self.debug) & (len(self.fit) > 5): + if (self.debug) & ( (len(self.fit) > 5) | (errors < 5) ): break except Exception as e: self._logger.critical(e, exc_info=True) + + errors += 1 self.fit = pd.DataFrame.from_records(self.fit) @@ -445,13 +449,15 @@ class ShearTestExtensionTUDresden(ShearTestExtension): self.unit_s = 1 / 1000. #mm self.unit_F = 1.0 #N - self.unit_t = 1.0 #s + self.unit_t = 1.0 #s def update_parameter(self): + + self.meta_names_of_parameter = { 't': ['TIME'], - 'speciment_diameter': ['Diameter of specimen\r\n'], + 'speciment_diameter': ['Diameter of specimen'], } #list of names self.data_column_names = {