From cd9bd088638bfcc061a0015f42a45ed232d66fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Clau=C3=9F?= Date: Fri, 16 Jun 2023 09:43:33 +0200 Subject: [PATCH] add regression citt, add new datafields to citt --- src/paveit/datamodels/__init__.py | 1 + src/paveit/datamodels/citt.py | 6 ++ src/paveit/datamodels/regression.py | 44 +++++++++ src/paveit/labtest/base.py | 18 +++- src/paveit/labtest/citt.py | 69 ++++++++------ src/paveit/postprocessing/__init__.py | 4 + src/paveit/postprocessing/citt.py | 131 ++++++++++++++++++++++++++ 7 files changed, 244 insertions(+), 29 deletions(-) create mode 100755 src/paveit/datamodels/regression.py create mode 100755 src/paveit/postprocessing/__init__.py create mode 100644 src/paveit/postprocessing/citt.py diff --git a/src/paveit/datamodels/__init__.py b/src/paveit/datamodels/__init__.py index 9e2768e..9604091 100755 --- a/src/paveit/datamodels/__init__.py +++ b/src/paveit/datamodels/__init__.py @@ -13,3 +13,4 @@ from .sheartest import * from .taskmanager import * from .usermanagement import * from .workpackage import * +from .regression import * \ No newline at end of file diff --git a/src/paveit/datamodels/citt.py b/src/paveit/datamodels/citt.py index 6db2ee3..01da2e5 100755 --- a/src/paveit/datamodels/citt.py +++ b/src/paveit/datamodels/citt.py @@ -26,6 +26,7 @@ class CyclicIndirectTensileTest(Document): filehash = StringField(required=True) + speciment_name = StringField(required=True, default=None) meta = { @@ -45,6 +46,9 @@ class CITTSiffnessResults(CyclicIndirectTensileTest): f_set = FloatField() sigma_set = FloatField() T_set = FloatField() + speciment_diameter = FloatField(required=True) + speciment_height = FloatField(required=True) + N_from = IntField() N_to = IntField() @@ -55,6 +59,8 @@ class CITTSiffnessResults(CyclicIndirectTensileTest): stiffness = FloatField() nu = FloatField() phase = FloatField() + el_strains = FloatField() + sigma_calc = FloatField() #required parameter ## F F_amp = FloatField() diff --git a/src/paveit/datamodels/regression.py b/src/paveit/datamodels/regression.py new file mode 100755 index 0000000..3af0e61 --- /dev/null +++ b/src/paveit/datamodels/regression.py @@ -0,0 +1,44 @@ +import datetime + +from mongoengine import * + +from .taskmanager import TaskManagerBase + +class RegressionBase(Document): + + date = DateTimeField(default=datetime.datetime.now(), + wtf_options={"render_kw": { + "step": "60" + }}) + + task_id = LazyReferenceField(TaskManagerBase, required=True) + + #statistische Werte + stat_r2 = FloatField(required=False) + + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'regression', + "db_alias": 'dblabtests', + } + +class RegCITT(RegressionBase): + + nsamples = IntField() + + Emax = FloatField(min_value=0, max_value=150000) + Emin = FloatField() + + T0 = FloatField(min_value=-100, max_value=100) + + phi = FloatField() + z0 = FloatField() + z1 = FloatField() + + + \ No newline at end of file diff --git a/src/paveit/labtest/base.py b/src/paveit/labtest/base.py index c1f65ef..94bbec5 100755 --- a/src/paveit/labtest/base.py +++ b/src/paveit/labtest/base.py @@ -163,7 +163,6 @@ class DataSineLoad(): self._logger.debug(f'columns: {colnames}') print(self.data.head()) - print(self.data['s_piston'].head()) self._logger.debug(f'standardize_data: {self.data.columns}') @@ -171,10 +170,11 @@ class DataSineLoad(): def _standardize_meta(self): self._logger.debug('run _standardize_meta') - # remove "\r\n" ending from Windows + # remove "\r\n" ending from Windows and whitespace for col in list(self.metadata.keys()): col_mod = col.replace('\r\n', '') + col_mod = col_mod.strip() if col != col_mod: self.metadata[col_mod] = self.metadata[col] @@ -189,6 +189,13 @@ class DataSineLoad(): self.metadata.pop(name) break + + # stip data + for key in self.metadata.keys(): + try: + self.metadata[key] = self.metadata[key].strip() + except: + pass self._logger.debug(f'meta (stand.): {self.metadata}') @@ -247,6 +254,13 @@ class DataSineLoad(): for col in ['time']: self.data[col] = self.data[col].mul(self.unit_t) + + try: + + self.data['f'] = self.data['f'].mul(self.unit_freq) + + except: + pass return True diff --git a/src/paveit/labtest/citt.py b/src/paveit/labtest/citt.py index 68b1e92..7671d65 100755 --- a/src/paveit/labtest/citt.py +++ b/src/paveit/labtest/citt.py @@ -54,6 +54,8 @@ class CITTBase(DataSineLoad): } def _sel_df(self, df, num=5, shift=-1): + + print(df.head()) N = df['N'].unique() n_N = len(N) @@ -61,7 +63,7 @@ class CITTBase(DataSineLoad): min_N = min(N) freq = float(df['f'].unique()[0]) - + # define cycles to select if freq == 10.0: Nfrom = 98 @@ -84,6 +86,10 @@ class CITTBase(DataSineLoad): else: Nfrom = None Nto = None + + + self._logger.debug(f'{min_N}, {max_N}, {n_N}, {num}, {shift}') + self._logger.debug(f'Frequenz: {freq}, Nfrom: {Nfrom}, Nto: {Nto}') # Fall 1: nur num Lastwechsel if n_N < num: @@ -229,18 +235,27 @@ class CITTBase(DataSineLoad): ## Stiffness deltaF = res_temp['fit_F_amp'] + deltaU = res_temp['fit_s_hor_sum_amp'] + + h = float(self.metadata['speciment_height']) + d = float(self.metadata['speciment_diameter']) + nu = calc_nu(temperature) res_temp['nu'] = nu - h = float(self.metadata['speciment_height']) - - deltaU = res_temp['fit_s_hor_sum_amp'] - - res_temp['stiffness'] = (deltaF * (0.274 + nu)) / (h * deltaU) + #nach TP Asphalt 26 + res_temp['stiffness'] = deltaF /(h * deltaU) (4.0/np.pi -1 + nu) + + ## Elastische hori. Dehnung + res_temp['el_strains'] = 2*deltaU/d * (1+3*nu)/(4 + np.pi*nu - np.pi) * 1000.0 + + ## maximale Zugspannung im Probekörpermittelpunkt + res_temp['sigma_calc'] = (2*deltaF)/(np.pi*d*h) + # TODO: Überarbeiten und erweitern (ISSUE #2) - res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[ - 'fit_s_hor_sum_phase'] + res_temp['phase'] = res_temp['fit_F_phase'] - res_temp['fit_s_hor_sum_phase'] + except Exception as e: self._logger.exception(e) res_temp = None @@ -284,6 +299,9 @@ class CITTBase(DataSineLoad): else: meta['speciment_name'] = self.filename + meta['speciment_diameter'] = self.metadata['speciment_diameter'] + meta['speciment_height'] = self.metadata['speciment_height'] + #check if result in db #n = CITTSiffness.objects(**meta).count() @@ -346,7 +364,7 @@ class CITT_TUDresdenWille(CITTBase): 't': ['Zeit'], 'speciment_diameter': ['Probekörberdurchmesser', 'Probekörberbreite'], 'speciment_height': ['Probekörperhöhe'], - 'speciment_name': ['Probekörper-Nummer Bezeichnung'], + 'speciment_name': ['Probekörper', 'Probekörper-Nummer Bezeichnung'], } #list of names self.data_column_names = { @@ -387,7 +405,7 @@ class CITT_TUDresdenGeosysOne(CITTBase): 't': ['Zeit'], 'speciment_diameter': ['Probekörberdurchmesser', 'Probekörberbreite'], 'speciment_height': ['Probekörperhöhe'], - 'speciment_name': ['Probekörper-Nummer Bezeichnung'], + 'speciment_name': ['Probekörper', 'Probekörper-Nummer Bezeichnung'], } #list of names self.data_column_names = { @@ -806,28 +824,31 @@ class CITT_LaborHart(CITTBase): def _define_units(self): - self.unit_s = 1.0 #mm - self.unit_F = 1.0 #N + self.unit_s = 1.0/1000.0/10.0 #mm 10fach überhöht abgelegt + self.unit_F = 1.0/10.0 #N 10fach überhöht abgelegt self.unit_t = 1. / 1000.0 #s + + self.unit_freq = 1.0/10. #s 10fach überhöht abgelegt def update_parameter(self): self.meta_names_of_parameter = { 'sigma': ['Oberspannung'], 'T': ['Solltemperatur'], - 't': ['TIME'], + 't': ['Time_ms'], 'speciment_diameter': ['Probendurchmesser'], 'speciment_height': ['Probenhöhe'], + 'speciment_name': ['Probenbezeichnung'], } #list of names self.data_column_names = { - 'time': ['TIME'], - 'f': ['FREQUENZ'], - 'F': ['Load'], - 's_hor_1': ['SENSOR 4'], - 's_hor_2': ['SENSOR Extension'], - 's_piston': ['Position'], - 'N': ['Impulsnummer'], + 'time': ['Time_ms'], + 'f': ['FREQUENZ_x10'], + 'F': ['Load_N_x10'], + 's_hor_1': ['SENSOR_EXT_µm_x10'], + 's_hor_2': ['SENSOR_S4_µm_x10'], + 's_piston': ['POSITION_µm'], + 'N': ['Zyklen_fortlaufend'], } def _process_data(self): @@ -884,13 +905,7 @@ class CITT_LaborHart(CITTBase): # FIX: Sigma nicht in Metadaten oder Messdaten enthalten print(meta) - if not "sigma" in self.metadata: - sigma = float( - os.path.split(self.filename)[-1].split('MPa')[0].strip().replace( - ',', '.')) - - meta['sigma'] = sigma - + #clean data data = data.dropna(axis=1) diff --git a/src/paveit/postprocessing/__init__.py b/src/paveit/postprocessing/__init__.py new file mode 100755 index 0000000..ff3c8b2 --- /dev/null +++ b/src/paveit/postprocessing/__init__.py @@ -0,0 +1,4 @@ +from .citt import citt + +__all__ = ['citt', + ] \ No newline at end of file diff --git a/src/paveit/postprocessing/citt.py b/src/paveit/postprocessing/citt.py new file mode 100644 index 0000000..555e38f --- /dev/null +++ b/src/paveit/postprocessing/citt.py @@ -0,0 +1,131 @@ +import datetime + +import numpy as np +from bson import ObjectId +from paveit.datamodels import CITTSiffnessResults, RegCITT +import lmfit as lm +import pandas as pd +import scipy.special as sf +from scipy.optimize import curve_fit + +def temp_freq_equivalence(T, f, phi, T0=20.0): + + alphaT = np.exp(phi * ((1 / (T + 273.15)) - (1 / (T0 + 273.15)))) + x = np.log(f * alphaT) / np.log(10) + + return x + +def stiffness_tp26(T, f, phi, Emax, Emin, z0, z1, T0=20.0): + + x = temp_freq_equivalence(T, f, phi, T0) + + E = Emin + (Emax - Emin) / (1 + np.exp(z1 * x + z0)) + + return E + + +def calc_nu(T): + #TODO: Prüfen ob Formel stimmt! + nu = 0.15 + (0.35) / (1 + np.exp(3.1849 - 0.04233 * (9 / 5 * T + 32))) + return nu + + +def citt(task_id: str): + """ + Postprocessing task + """ + + print('postprocessing') + + task_id = ObjectId(task_id) + + # read all data + data = [] + parlist = ['f_set', 'T_set', 'stiffness', 'phase'] + + for obj in CITTSiffnessResults.objects(task_id=task_id).only(*parlist): + data.append(dict((k, obj[k]) for k in parlist )) + + data = pd.DataFrame.from_records(data) + + #Emax/Emin + line_mod = lm.models.LinearModel() + out = line_mod.fit(data.stiffness, x=data.phase) + + print(out.best_values) + + Emax = line_mod.eval(out.params, x=0.0) + + Emin = 0 + + assert Emin < Emax + + print(data.head()) + + + # Fit data + mod = lm.models.Model(stiffness_tp26, independent_vars=['f','T']) + + mod.set_param_hint( + 'Emin', + value=Emin, + min=0, + max=0.9*Emax, + vary=True, + ) + + mod.set_param_hint( + 'Emax', + value=Emax, + min=0.9*Emax, + max=1.1*Emax, + vary=True, + ) + + mod.set_param_hint( + 'T0', + value=20.0, + vary=False, + ) + + mod.set_param_hint('phi', value=25000, min=15000, max=35000, vary=True) + mod.set_param_hint('z0', value=1,min=1e-10, max=1000., vary=True) + mod.set_param_hint('z1', value=-1, min=-1000., max=-1e-10, vary=True) + + parms_fit = [ + mod.param_hints['Emin']['value'], mod.param_hints['Emax']['value'], + mod.param_hints['phi']['value'], mod.param_hints['z0']['value'], + mod.param_hints['z1']['value'] + ] + + + ## run fit + results = [] + r2 = [] + + methods = ['leastsq', 'powell'] + + for method in methods: + result = mod.fit(data.stiffness, T=data.T_set, f=data.f_set, method=method, verbose=False) + + r2temp = 1.0 - result.redchi / np.var(data.stiffness.values, ddof=2) + r2.append(r2temp) + + results.append(result) + + best = np.nanargmax(r2) + + res = results[best].best_values + res['nsamples'] = len(data) + res['task_id'] = task_id + + res['stat_r2'] = r2[best] + + res['date'] = datetime.datetime.now() + + print(res) + + # save results to db + doc = RegCITT.objects(task_id=task_id).modify(upsert=True, **res) + + return True \ No newline at end of file