add regression citt, add new datafields to citt

src/paveit/datamodels/__init__.py

@@ -13,3 +13,4 @@ from .sheartest import *
 from .taskmanager import *
 from .usermanagement import *
 from .workpackage import *
+from .regression import *

src/paveit/datamodels/citt.py

@@ -26,6 +26,7 @@ class CyclicIndirectTensileTest(Document):
 
 
     filehash = StringField(required=True)
+    
     speciment_name = StringField(required=True, default=None)
 
     meta = {
@@ -45,6 +46,9 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
     f_set = FloatField()
     sigma_set = FloatField()
     T_set = FloatField()
+    speciment_diameter = FloatField(required=True)    
+    speciment_height = FloatField(required=True)
+
 
     N_from = IntField()
     N_to = IntField()
@@ -55,6 +59,8 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
     stiffness = FloatField()
     nu = FloatField()
     phase = FloatField()
+    el_strains = FloatField()
+    sigma_calc = FloatField()
     #required parameter
     ## F
     F_amp = FloatField()

src/paveit/datamodels/regression.py

@@ -0,0 +1,44 @@
+import datetime
+
+from mongoengine import *
+
+from .taskmanager import TaskManagerBase
+
+class RegressionBase(Document):
+
+    date = DateTimeField(default=datetime.datetime.now(),
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+    
+    task_id = LazyReferenceField(TaskManagerBase, required=True)
+    
+    #statistische Werte
+    stat_r2 = FloatField(required=False)
+    
+
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'regression',
+        "db_alias": 'dblabtests',
+    }
+    
+class RegCITT(RegressionBase):
+    
+    nsamples = IntField()
+
+    Emax = FloatField(min_value=0, max_value=150000)
+    Emin = FloatField()
+    
+    T0 = FloatField(min_value=-100, max_value=100)
+    
+    phi = FloatField()
+    z0 = FloatField()
+    z1 = FloatField()
+    
+    
+    

src/paveit/labtest/base.py

@@ -163,7 +163,6 @@ class DataSineLoad():
         self._logger.debug(f'columns: {colnames}')
         
         print(self.data.head())
-        print(self.data['s_piston'].head())
         
         self._logger.debug(f'standardize_data: {self.data.columns}')
         
@@ -171,10 +170,11 @@ class DataSineLoad():
     def _standardize_meta(self):
         self._logger.debug('run _standardize_meta')
         
-        # remove "\r\n" ending from Windows
+        # remove "\r\n" ending from Windows and whitespace
         for col in list(self.metadata.keys()):
             
             col_mod = col.replace('\r\n', '')
+            col_mod = col_mod.strip()
             
             if col != col_mod:
                 self.metadata[col_mod] = self.metadata[col]
@@ -190,6 +190,13 @@ class DataSineLoad():
 
                     break
                 
+        # stip data
+        for key in self.metadata.keys():
+            try:                
+                self.metadata[key] = self.metadata[key].strip()
+            except:
+                pass
+        
         self._logger.debug(f'meta (stand.): {self.metadata}')
 
     def _modify_meta(self):
@@ -248,6 +255,13 @@ class DataSineLoad():
         for col in ['time']:
             self.data[col] = self.data[col].mul(self.unit_t)
             
+        try:
+            
+            self.data['f'] = self.data['f'].mul(self.unit_freq)
+            
+        except:
+            pass
+
         return True
 
     def _post_round_values(self):

src/paveit/labtest/citt.py

@@ -55,6 +55,8 @@ class CITTBase(DataSineLoad):
 
     def _sel_df(self, df, num=5, shift=-1):
         
+        print(df.head())
+
         N = df['N'].unique()
         n_N = len(N)
         max_N = max(N)
@@ -85,6 +87,10 @@ class CITTBase(DataSineLoad):
             Nfrom = None
             Nto = None
             
+            
+        self._logger.debug(f'{min_N}, {max_N}, {n_N}, {num}, {shift}')
+        self._logger.debug(f'Frequenz: {freq}, Nfrom: {Nfrom}, Nto: {Nto}')
+
         # Fall 1: nur num Lastwechsel
         if n_N < num:
             df_sel = None
@@ -229,18 +235,27 @@ class CITTBase(DataSineLoad):
 
                 ## Stiffness
                 deltaF = res_temp['fit_F_amp']
+                deltaU = res_temp['fit_s_hor_sum_amp']
+                
+                h = float(self.metadata['speciment_height'])
+                d = float(self.metadata['speciment_diameter'])
+                
                 nu = calc_nu(temperature)
                 res_temp['nu'] = nu
 
-                h = float(self.metadata['speciment_height'])
+                #nach TP Asphalt 26
+                res_temp['stiffness'] = deltaF /(h * deltaU)  (4.0/np.pi -1 + nu)
                 
-                deltaU = res_temp['fit_s_hor_sum_amp']
+                ## Elastische hori. Dehnung                
+                res_temp['el_strains'] = 2*deltaU/d * (1+3*nu)/(4 + np.pi*nu - np.pi) * 1000.0
+                
+                ## maximale Zugspannung im Probekörpermittelpunkt
+                res_temp['sigma_calc'] = (2*deltaF)/(np.pi*d*h)
                 
-                res_temp['stiffness'] = (deltaF * (0.274 + nu)) / (h * deltaU)
 
                 # TODO: Überarbeiten und erweitern (ISSUE #2)
-                res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[
+                res_temp['phase'] = res_temp['fit_F_phase'] - res_temp['fit_s_hor_sum_phase']
-                    'fit_s_hor_sum_phase']
+                
             except Exception as e:
                 self._logger.exception(e)
                 res_temp = None
@@ -284,6 +299,9 @@ class CITTBase(DataSineLoad):
             else:
                 meta['speciment_name'] = self.filename
             
+            meta['speciment_diameter'] = self.metadata['speciment_diameter']
+            meta['speciment_height'] = self.metadata['speciment_height']
+            
 
             #check if result in db
             #n = CITTSiffness.objects(**meta).count()
@@ -346,7 +364,7 @@ class CITT_TUDresdenWille(CITTBase):
             't': ['Zeit'],
             'speciment_diameter': ['Probekörberdurchmesser', 'Probekörberbreite'],
             'speciment_height': ['Probekörperhöhe'],
-            'speciment_name': ['Probekörper-Nummer Bezeichnung'],
+            'speciment_name': ['Probekörper', 'Probekörper-Nummer Bezeichnung'],
         }  #list of names
 
         self.data_column_names = {
@@ -387,7 +405,7 @@ class CITT_TUDresdenGeosysOne(CITTBase):
             't': ['Zeit'],
             'speciment_diameter': ['Probekörberdurchmesser', 'Probekörberbreite'],
             'speciment_height': ['Probekörperhöhe'],
-            'speciment_name': ['Probekörper-Nummer Bezeichnung'],
+            'speciment_name': ['Probekörper', 'Probekörper-Nummer Bezeichnung'],
         }  #list of names
 
         self.data_column_names = {
@@ -806,28 +824,31 @@ class CITT_LaborHart(CITTBase):
 
     def _define_units(self):
 
-        self.unit_s = 1.0  #mm
+        self.unit_s = 1.0/1000.0/10.0  #mm 10fach überhöht abgelegt
-        self.unit_F = 1.0  #N
+        self.unit_F = 1.0/10.0  #N  10fach überhöht abgelegt
         self.unit_t = 1. / 1000.0  #s
         
+        self.unit_freq = 1.0/10.  #s  10fach überhöht abgelegt
+
     def update_parameter(self):
 
         self.meta_names_of_parameter = {
             'sigma': ['Oberspannung'],
             'T': ['Solltemperatur'],
-            't': ['TIME'],
+            't': ['Time_ms'],
             'speciment_diameter': ['Probendurchmesser'],
             'speciment_height': ['Probenhöhe'],
+            'speciment_name': ['Probenbezeichnung'],
         }  #list of names
 
         self.data_column_names = {
-            'time': ['TIME'],
+            'time': ['Time_ms'],
-            'f': ['FREQUENZ'],
+            'f': ['FREQUENZ_x10'],
-            'F': ['Load'],
+            'F': ['Load_N_x10'],
-            's_hor_1': ['SENSOR 4'],
+            's_hor_1': ['SENSOR_EXT_µm_x10'],
-            's_hor_2': ['SENSOR Extension'],
+            's_hor_2': ['SENSOR_S4_µm_x10'],
-            's_piston': ['Position'],
+            's_piston': ['POSITION_µm'],
-            'N': ['Impulsnummer'],
+            'N': ['Zyklen_fortlaufend'],
         }
 
     def _process_data(self):
@@ -884,12 +905,6 @@ class CITT_LaborHart(CITTBase):
 
         # FIX: Sigma nicht in Metadaten oder Messdaten enthalten
         print(meta)
-        if not "sigma" in self.metadata:
-            sigma = float(
-                os.path.split(self.filename)[-1].split('MPa')[0].strip().replace(
-                    ',', '.'))
-
-            meta['sigma'] = sigma
         
         #clean data
         data = data.dropna(axis=1)

src/paveit/postprocessing/__init__.py

@@ -0,0 +1,4 @@
+from .citt import citt
+
+__all__ = ['citt',
+           ]

src/paveit/postprocessing/citt.py

@@ -0,0 +1,131 @@
+import datetime
+
+import numpy as np
+from bson import ObjectId
+from paveit.datamodels import CITTSiffnessResults, RegCITT
+import lmfit as lm
+import pandas as pd
+import scipy.special as sf
+from scipy.optimize import curve_fit
+
+def temp_freq_equivalence(T, f, phi, T0=20.0):
+    
+    alphaT = np.exp(phi * ((1 / (T + 273.15)) - (1 / (T0 + 273.15))))
+    x = np.log(f * alphaT) / np.log(10)
+    
+    return x
+
+def stiffness_tp26(T, f, phi, Emax, Emin, z0, z1, T0=20.0):
+
+    x = temp_freq_equivalence(T, f, phi, T0)    
+    
+    E = Emin + (Emax - Emin) / (1 + np.exp(z1 * x + z0))
+
+    return E
+
+
+def calc_nu(T):
+    #TODO: Prüfen ob Formel stimmt!
+    nu = 0.15 + (0.35) / (1 + np.exp(3.1849 - 0.04233 * (9 / 5 * T + 32)))
+    return nu
+
+
+def citt(task_id: str):
+    """
+    Postprocessing task
+    """
+    
+    print('postprocessing')
+    
+    task_id = ObjectId(task_id)
+    
+    # read all data
+    data = []
+    parlist = ['f_set', 'T_set', 'stiffness', 'phase']
+    
+    for obj in CITTSiffnessResults.objects(task_id=task_id).only(*parlist):
+        data.append(dict((k, obj[k]) for k in parlist ))
+        
+    data = pd.DataFrame.from_records(data)
+    
+    #Emax/Emin
+    line_mod = lm.models.LinearModel()
+    out = line_mod.fit(data.stiffness, x=data.phase)
+    
+    print(out.best_values)
+    
+    Emax = line_mod.eval(out.params, x=0.0)
+    
+    Emin = 0
+    
+    assert Emin < Emax
+    
+    print(data.head())
+    
+    
+    # Fit data        
+    mod = lm.models.Model(stiffness_tp26, independent_vars=['f','T'])
+    
+    mod.set_param_hint(
+        'Emin',
+        value=Emin,
+        min=0,
+        max=0.9*Emax,
+        vary=True,
+    )
+    
+    mod.set_param_hint(
+        'Emax',
+        value=Emax,
+        min=0.9*Emax,
+        max=1.1*Emax,
+        vary=True,
+    )
+    
+    mod.set_param_hint(
+        'T0',
+        value=20.0,
+        vary=False,
+    )
+    
+    mod.set_param_hint('phi', value=25000, min=15000, max=35000, vary=True)
+    mod.set_param_hint('z0', value=1,min=1e-10, max=1000., vary=True)
+    mod.set_param_hint('z1', value=-1, min=-1000., max=-1e-10, vary=True)
+    
+    parms_fit = [
+        mod.param_hints['Emin']['value'], mod.param_hints['Emax']['value'],
+        mod.param_hints['phi']['value'], mod.param_hints['z0']['value'],
+        mod.param_hints['z1']['value']
+    ]
+    
+    
+    ## run fit
+    results = []
+    r2 = []
+
+    methods = ['leastsq', 'powell']
+
+    for method in methods:
+        result = mod.fit(data.stiffness, T=data.T_set, f=data.f_set, method=method, verbose=False)
+        
+        r2temp =  1.0 - result.redchi / np.var(data.stiffness.values, ddof=2)
+        r2.append(r2temp)
+
+        results.append(result)
+
+    best = np.nanargmax(r2)
+    
+    res = results[best].best_values
+    res['nsamples'] = len(data)
+    res['task_id'] = task_id
+    
+    res['stat_r2'] = r2[best]
+    
+    res['date'] = datetime.datetime.now()
+
+    print(res)
+    
+    # save results to db
+    doc = RegCITT.objects(task_id=task_id).modify(upsert=True, **res)
+    
+    return True