Datenmodelle angepasst, einige Firmen in CITT übernommen

src/paveit/datamodels/__init__.py

@@ -0,0 +1,7 @@
+from .citt import *
+from .data import *
+from .material import *
+from .project import *
+from .sheartest import *
+from .usermanagement import *
+from .workpackage import *

src/paveit/datamodels/citt.py

@@ -0,0 +1,102 @@
+import datetime
+
+from mongoengine import *
+
+from .material import Material
+from .project import Project
+from .usermanagement import Organisation, User
+from .workpackage import Workpackage
+
+
+class CyclicIndirectTensileTest(Document):
+
+    date = DateTimeField(default=datetime.datetime.now,
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+
+    standard = StringField(default='TP Asphalt Teil 24')
+
+    org_id = LazyReferenceField(Organisation, required=True)
+    project_id = LazyReferenceField(Project, required=True)
+    workpackage_id = LazyReferenceField(Workpackage, required=False)
+
+    user_id = LazyReferenceField(User,
+                                 required=True,
+                                 reverse_delete_rule=DO_NOTHING)
+
+    material = LazyReferenceField(Material, required=True)
+
+    tags = ListField(StringField())
+
+    machine = StringField(default=None)
+
+    filehash = StringField(required=True)
+    speciment_name = StringField()
+
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'citt',
+        "db_alias": 'dblabtests',
+    }
+
+
+class CITTSiffnessResults(CyclicIndirectTensileTest):
+
+    #metadata
+    f_set = FloatField()
+    sigma_set = FloatField()
+    T_set = FloatField()
+
+    N_from = IntField()
+    N_to = IntField()
+    N_tot = IntField()
+    n_samples_per_cycle = IntField()
+
+    #results
+    stiffness = FloatField()
+    nu = FloatField()
+    phase = FloatField()
+    #required parameter
+    ## F
+    F_amp = FloatField()
+    F_freq = FloatField()
+    F_phase = FloatField()
+    F_offset = FloatField()
+    F_slope = FloatField()
+    F_r2 = FloatField()
+    F_min = FloatField()
+    F_max = FloatField()
+    ## S1
+    s_hor_1_amp = FloatField()
+    s_hor_1_freq = FloatField()
+    s_hor_1_phase = FloatField()
+    s_hor_1_offset = FloatField()
+    s_hor_1_slope = FloatField()
+    s_hor_1_r2 = FloatField()
+    s_hor_1_min = FloatField()
+    s_hor_1_max = FloatField()
+    ## S2
+    s_hor_2_amp = FloatField()
+    s_hor_2_freq = FloatField()
+    s_hor_2_phase = FloatField()
+    s_hor_2_offset = FloatField()
+    s_hor_2_slope = FloatField()
+    s_hor_2_r2 = FloatField()
+    s_hor_2_min = FloatField()
+    s_hor_2_max = FloatField()
+    ## S-Sum
+    s_hor_sum_amp = FloatField()
+    s_hor_sum_freq = FloatField()
+    s_hor_sum_phase = FloatField()
+    s_hor_sum_offset = FloatField()
+    s_hor_sum_slope = FloatField()
+    s_hor_sum_r2 = FloatField()
+    s_hor_sum_min = FloatField()
+    s_hor_sum_max = FloatField()
+
+    #optional parameter

src/paveit/datamodels/data.py

@@ -0,0 +1,57 @@
+import datetime
+
+from mongoengine import *
+
+from .citt import CyclicIndirectTensileTest
+from .sheartest import DynamicShearTest
+
+
+class RawData(Document):
+
+    date = DateTimeField(default=datetime.datetime.now,
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'rawdata',
+        "db_alias": 'dblabtests',
+    }
+
+
+class DataSheartest(RawData):
+
+    #results
+    result_id = LazyReferenceField(DynamicShearTest,
+                                   required=True,
+                                   reverse_delete_rule=CASCADE)
+
+    # data
+    time = ListField(FloatField())
+    F = ListField(FloatField())
+    N = ListField(IntField())
+    s_vert_1 = ListField(FloatField())
+    s_vert_2 = ListField(FloatField())
+    s_vert_sum = ListField(FloatField(), required=False)
+    s_piston = ListField(FloatField(), required=False)
+
+
+class CITTSiffness(RawData):
+
+    result = LazyReferenceField(CyclicIndirectTensileTest,
+                                required=True,
+                                reverse_delete_rule=CASCADE)
+
+    # data
+    time = ListField(FloatField())
+    F = ListField(FloatField())
+    N = ListField(IntField())
+    s_hor_1 = ListField(FloatField())
+    s_hor_2 = ListField(FloatField())
+    s_hor_sum = ListField(FloatField())
+    s_piston = ListField(FloatField(), required=False)

src/paveit/datamodels/material.py

@@ -0,0 +1,89 @@
+import datetime
+from re import T
+
+from mongoengine import *
+
+from .project import Project
+from .usermanagement import Organisation, User
+
+
+class Material(Document):
+
+    date = DateTimeField(default=datetime.datetime.now,
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+
+    org_id = LazyReferenceField(Organisation,
+                                required=True,
+                                reverse_delete_rule=CASCADE)
+
+    project_id = ListField(LazyReferenceField(Project,
+                                              required=False,
+                                              reverse_delete_rule=CASCADE),
+                           required=True)
+
+    user_id = LazyReferenceField(User,
+                                 required=False,
+                                 reverse_delete_rule=DO_NOTHING)
+
+    tags = ListField(StringField())
+    norm = StringField(required=True, default='TP Asphalt Teil 24')
+
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'materials',
+        'indexes': [
+            [("material", 1)],
+            [("name", 1)],
+        ]
+    }
+
+
+class Asphalt(Material):
+
+    name = StringField()
+    material = StringField()
+    bitumen = StringField(required=False)
+
+    young_modulus = DictField()
+    fatigue = DictField()
+
+
+class Bitumen(Material):
+
+    name = StringField()
+    material = StringField()
+
+    young_modulus = DictField()
+
+
+class Bitumenemulsion(Material):
+
+    name = StringField()
+    material = StringField()
+
+    young_modulus = DictField()
+
+
+class Epoxy(Material):
+    name = StringField()
+    material = StringField()
+
+    young_modulus = DictField()
+
+
+class Kompaktasphalt(Material):
+    name = StringField()
+
+
+class Dummy(Material):
+
+    name = StringField()
+    material = StringField()
+
+    young_modulus = DictField()

src/paveit/datamodels/project.py

@@ -0,0 +1,42 @@
+import datetime
+
+from mongoengine import *
+
+from .usermanagement import Organisation, User
+
+
+class Project(Document):
+
+    name = StringField(required=True)
+    name_short = StringField(required=False)
+    project_id = StringField(required=True)
+    client = StringField(required=False)
+
+    date = DateTimeField(default=datetime.datetime.now,
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+
+    org_id = LazyReferenceField(Organisation,
+                                required=True,
+                                reverse_delete_rule=CASCADE)
+
+    user_id = LazyReferenceField(User,
+                                 required=False,
+                                 reverse_delete_rule=DO_NOTHING)
+
+    tags = ListField(StringField())
+
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'projects',
+        'indexes': [
+            [("name_short", 1)],
+            [("name", 1)],
+            [("project_id", 1)],
+        ]
+    }

src/paveit/datamodels/sheartest.py

@@ -0,0 +1,107 @@
+import datetime
+
+from mongoengine import *
+
+from .material import Material
+from .project import Project
+from .usermanagement import Organisation, User
+from .workpackage import Workpackage
+
+
+class DynamicShearTest(Document):
+
+    date = DateTimeField(default=datetime.datetime.now,
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+
+    org_id = LazyReferenceField(Organisation, required=True)
+    project_id = LazyReferenceField(Project, required=True)
+    workpackage_id = LazyReferenceField(Workpackage, required=False)
+
+    user_id = LazyReferenceField(User,
+                                 required=True,
+                                 reverse_delete_rule=DO_NOTHING)
+
+    material1 = LazyReferenceField(Material, required=True)
+    material2 = LazyReferenceField(Material, required=True)
+    bounding = LazyReferenceField(Material, required=True)
+
+    gap_width = FloatField(default=1.0)
+
+    tags = ListField(StringField())
+
+    standard = StringField(default='TP Asphalt Teil 24')
+
+    machine = StringField(default=None)
+
+    filehash = StringField(required=True)
+    speciment_name = StringField()
+
+    meta = {
+        'allow_inheritance':
+        True,
+        'index_opts': {},
+        'index_background':
+        True,
+        'index_cls':
+        False,
+        'auto_create_index':
+        True,
+        'collection':
+        'sheartest',
+        'indexes': [
+            [("lab", 1)],
+            [("speciment_name", 1)],
+            [("project", 1)],
+            [("bruch", 1)],
+            [("lab", 1), ("project", 1)],
+            [("lab", 1), ("project", 1), ("workpackage", 1)],
+            [("lab", 1), ("project", 1), ("bounding", 1)],
+        ]
+    }
+
+
+class DynamicShearTestExtension(DynamicShearTest):
+
+    #metadata
+    f = FloatField(required=True)
+    sigma_normal = FloatField(required=True)
+    T = FloatField(required=True)
+    extension = FloatField(required=True)
+
+    stiffness = FloatField(required=True)
+    bruch = BooleanField(required=True)
+    #fit parameter
+    ## F
+    fit_amp_F = FloatField(required=True)
+    fit_freq_F = FloatField(required=True)
+    fit_phase_F = FloatField(required=True)
+    fit_offset_F = FloatField(required=True)
+    fit_slope_F = FloatField(required=True)
+    ## S1
+    fit_amp_s_vert_1 = FloatField(required=True)
+    fit_freq_s_vert_1 = FloatField(required=True)
+    fit_phase_s_vert_1 = FloatField(required=True)
+    fit_offset_s_vert_1 = FloatField(required=True)
+    fit_slope_s_vert_1 = FloatField(required=True)
+    r2_s_vert_1 = FloatField(required=True)
+    ## S2
+    fit_amp_s_vert_2 = FloatField(required=True)
+    fit_freq_s_vert_2 = FloatField(required=True)
+    fit_phase_s_vert_2 = FloatField(required=True)
+    fit_offset_s_vert_2 = FloatField(required=True)
+    fit_slope_s_vert_2 = FloatField(required=True)
+    r2_s_vert_2 = FloatField(required=True)
+    ## S-Sum
+    fit_amp_s_vert_sum = FloatField(required=True)
+    fit_freq_s_vert_sum = FloatField(required=True)
+    fit_phase_s_vert_sum = FloatField(required=True)
+    fit_offset_s_vert_sum = FloatField(required=True)
+    fit_slope_s_vert_sum = FloatField(required=True)
+    r2_s_vert_sum = FloatField(required=True)
+    ## r2
+    r2_F = FloatField(required=True)
+    r2_s_vert_1 = FloatField(required=True)
+    r2_s_vert_2 = FloatField(required=True)
+    r2_s_vert_sum = FloatField(required=True)

src/paveit/datamodels/usermanagement.py

@@ -0,0 +1,51 @@
+import datetime
+
+from mongoengine import *
+
+
+class Organisation(Document):
+
+    name = StringField(required=True)
+    name_short = StringField(required=True)
+
+    date = DateTimeField(default=datetime.datetime.now,
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+
+    labtest_citt = StringField(required=False)
+
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'organisation',
+        'db_alias': 'dbusers',
+    }
+
+
+class User(Document):
+
+    active = BooleanField(required=True, default=True)
+
+    org_id = LazyReferenceField(Organisation,
+                                required=True,
+                                reverse_delete_rule=CASCADE)
+
+    date_added = DateTimeField(default=datetime.datetime.now,
+                               wtf_options={"render_kw": {
+                                   "step": "60"
+                               }})
+
+    name = StringField(required=True)
+
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'user'
+    }

src/paveit/datamodels/workpackage.py

@@ -0,0 +1,33 @@
+import datetime
+
+from mongoengine import *
+
+from .project import Project
+from .usermanagement import User
+
+
+class Workpackage(Document):
+
+    name = StringField(required=True)
+    name_short = StringField(required=False)
+    wp_id = StringField(required=True)
+
+    project_id = LazyReferenceField(Project, required=True)
+
+    user_id = LazyReferenceField(User,
+                                 required=False,
+                                 reverse_delete_rule=DO_NOTHING)
+
+    date = DateTimeField(default=datetime.datetime.now,
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'workpackages'
+    }

src/paveit/helper/__init__.py

@@ -1,8 +1,10 @@
 from .filehandling import read_file_to_bytesio
 from .filehasher import calc_hash_of_bytes
 from .minio import get_minio_client_archive, get_minio_client_processing
+from .mongo import connect_mongo_dbs, mongo_get_results
 
-__all__ = ['read_file_to_bytesio',
+__all__ = [
-               'get_minio_client_archive', 'get_minio_client_processing',
+    'read_file_to_bytesio', 'connect_mongo_dbs', 'mongo_get_results',
+    'get_minio_client_processing', 'get_minio_client_archive',
     'calc_hash_of_bytes'
-           ]
+]

src/paveit/helper/mongo.py

@@ -0,0 +1,97 @@
+from bson import ObjectId
+from mongoengine import connect as mongo_connect
+from pandas import DataFrame
+
+
+def connect_mongo_db_labtests(username='admin',
+                              password='admin',
+                              host='mongo',
+                              authentication_source='admin'):
+
+    mongo_connect('labtests',
+                  username=username,
+                  password=password,
+                  host=host,
+                  authentication_source=authentication_source,
+                  alias='dblabtests')
+
+
+def connect_mongo_db_usser(username='admin',
+                           password='admin',
+                           host='mongo',
+                           authentication_source='admin'):
+
+    mongo_connect('users',
+                  username=username,
+                  password=password,
+                  host=host,
+                  authentication_source=authentication_source,
+                  alias='dbusers')
+
+
+def connect_mongo_dbs(username='admin',
+                      password='admin',
+                      host='mongo',
+                      authentication_source='admin'):
+
+    connect_mongo_db_labtests(username=username,
+                              password=password,
+                              host=host,
+                              authentication_source=authentication_source)
+
+    connect_mongo_db_usser(username=username,
+                           password=password,
+                           host=host,
+                           authentication_source=authentication_source)
+
+
+def mongo_upload_results(resultsmodel, results: DataFrame, datamodel,
+                         data: DataFrame, filehash: str, org_id: ObjectId,
+                         project_id: ObjectId, material_id: ObjectId,
+                         user_id: ObjectId):
+
+    for idx, res in results.iterrows():
+
+        #upload results
+        meta['filehash'] = filehash
+        meta['org_id'] = org_id
+        meta['project_id'] = project_id
+        meta['material'] = material_id
+        meta['user_id'] = user_id
+
+        #check if result in db
+        #n = CITTSiffness.objects(**meta).count()
+        #print(n)
+
+        # write data
+        data_dict = res.to_dict()
+        data_dict.update(meta)
+
+        f = resultsmodel(**data_dict).save()
+
+        # upload data
+        data_sel = data[idx_fit]
+
+        # required data
+        data_out = dict(
+            time=data_sel.index,
+            F=list(data_sel['F']),
+            N=list(data_sel['N']),
+            s_hor_1=list(data_sel['s_hor_1']),
+            s_hor_2=list(data_sel['s_hor_2']),
+            s_hor_sum=list(data_sel['s_hor_sum']),
+        )
+        #optional data
+        for col in ['S_piston']:
+            if col in data_sel.columns:
+                data_out[col] = data_sel[col]
+
+        g = datamodel(result=f.id, **data_out).save()
+
+
+def mongo_get_results(resultsmodel, results: DataFrame, datamodel,
+                      data: DataFrame, filehash: str, org_id: ObjectId,
+                      project_id: ObjectId, material_id: ObjectId,
+                      user_id: ObjectId):
+
+    return True

src/paveit/io/__init__.py

@@ -0,0 +1,3 @@
+from .geosys import read_geosys
+
+__all__ = ["read_geosys"]

src/paveit/io/geosys.py

@@ -0,0 +1,227 @@
+import csv
+import os
+from io import BytesIO
+from sys import getsizeof
+
+from numpy import array
+from pandas import DataFrame
+
+
+def detect_tabnum(filename, tabstr, encoding='utf-8'):
+    filename = os.path.normpath(filename)
+
+    tabstr = tabstr.lower()
+
+    #Einlesen
+    with open(filename, 'r', encoding=encoding) as inFile:
+        reader = csv.reader(inFile, delimiter='\t')
+        counter = 0
+        for row in reader:
+
+            row = [r.lower() for r in row]
+            if any(tabstr in mystring for mystring in row):
+                if 'plain' in row:
+                    return row[1]
+
+            counter += 1
+
+            if counter > 100:
+                return False
+
+
+def str2float(str):
+    try:
+        str = str.replace(',', '.')
+        return float(str)
+    except:
+        return None
+
+
+def read_geosys(buffer: BytesIO,
+                table,
+                pkdata='001',
+                metadata_ids=['003', '015'],
+                encoding='utf-8',
+                to_si=False,
+                debug=False):
+    '''
+
+    :param buffer: Bytes IO Object
+    :param table: Table-Number
+    :param pkdata: Table-Number of speciment definitions, default: 1
+    :param encoding: Encoding, default: utf-8
+    :param debug: debug-mode
+    :return:
+
+    '''
+
+    try:
+        dictOut = {}
+        dictOut['durch'] = 0
+        dictOut['hoehe'] = 0
+
+        #---------------------------------------------------------------------
+        #Daten einlesen und umwandeln
+        #---------------------------------------------------------------------
+
+        data = []
+
+        #Einlesen
+        buffer.seek(0)
+
+        lines_file = buffer.readlines()
+
+        for line in lines_file:
+            try:
+                line = line.decode(encoding)
+                line = line.split('\t')
+
+                if len(line) > 2:
+                    data.append(line)
+
+            except:
+                pass
+
+        if debug:
+            print('Anz. Datensätze: ', str(len(data)), getsizeof(data))
+        #aufräumen
+        ##Datenstruktur anlegen
+
+        data_processed = {}
+        data_processed['head'] = []
+        data_processed['metadata'] = {}
+        data_processed['data'] = []
+
+        for i in metadata_ids:
+            data_processed['metadata'][i] = []
+
+        for idx, d in enumerate(data):
+            try:
+                v = d[0][0:3]
+                if v in pkdata: data_processed['head'].append(d)
+                if v in metadata_ids: data_processed['metadata'][v].append(d)
+                if v in table: data_processed['data'].append(d)
+
+            except:
+                pass
+
+        # replace object
+        data = data_processed
+
+        if debug:
+            print('data_clean fin')
+
+        ## Header aufbereiten
+
+        for idx, row in enumerate(data['head']):
+            if idx == 0:
+                id_durchmesser = None
+                id_hoehe = None
+                id_name = None
+
+                for idx_name, name in enumerate(row):
+                    name_lower = name.lower()
+
+                    if any(map(name_lower.__contains__, ['durchmesser'])):
+                        id_durchmesser = idx_name
+
+                    elif any(map(name_lower.__contains__, ['bezeichnung'])):
+                        id_name = idx_name
+
+                    elif any(map(name_lower.__contains__, ['höhe'])):
+                        id_hoehe = idx_name
+
+                if debug:
+                    print(id_durchmesser, id_hoehe, id_name)
+            elif idx == 1:
+                unit_durch = None
+                unit_hoehe = None
+
+                try:
+                    unit_durch = row[id_durchmesser]
+                    unit_hoehe = row[id_hoehe]
+                except:
+                    pass
+
+            elif idx == 2:
+                durchmesser = None
+                hoehe = None
+                name = None
+                try:
+                    durchmesser = str2float(row[id_durchmesser])
+                    hoehe = str2float(row[id_hoehe])
+                    name = row[id_name]
+
+                except:
+                    pass
+
+        header = {
+            'speciment_diameter': durchmesser,
+            'speciment_height': hoehe,
+            'name': name,
+            'unit_h': unit_hoehe,
+            'unit_d': unit_durch
+        }
+
+        meta = data['metadata']
+
+        for key in meta.keys():
+            sel = meta[key]
+
+            assert len(sel[0]) == len(sel[2])
+            header_append = {
+                sel[0][i]: sel[2][i].strip()
+                for i in range(len(sel[0]))
+            }
+
+            header.update(header_append)
+
+        if debug:
+            print('header\n', header)
+
+        # add metadata to header
+
+        ## Daten in Pandas DataFrame umwandeln
+        if debug:
+            print('daten umwandel')
+
+        temp = []
+        for idx, row in enumerate(data['data']):
+            if idx == 0:
+                if debug:
+                    print('head')
+                data_head = []
+                for idx_name, name in enumerate(row):
+                    if idx_name <= 1: continue
+                    data_head.append(name)
+            elif idx == 1:
+                data_units = []
+                for idx_name, name in enumerate(row):
+                    if idx_name <= 1: continue
+                    data_units.append(name)
+            else:
+                t = []
+                for idx_col, value in enumerate(row):
+                    if idx_col <= 1:
+                        continue
+                    else:
+                        t.append(str2float(value))
+
+                temp.append(t)
+
+        data = array(temp)
+
+        if debug:
+            print(data_head, data_units)
+
+        ## Bezeichnungen der Daten normalisieren
+        # Pandas DataFrame erstellen
+        data = DataFrame(data=data, columns=data_head)
+        if debug:
+            print(data.head())
+
+        return header, data
+
+    except:
+        print('Fehler beim lesen')
+        raise

src/paveit/labtest/base.py

@@ -40,6 +40,12 @@ class DataSineLoad():
 
         self._pre_run()
 
+    def _which_machine(self):
+        """
+        check the file and try to get the machine from the data
+        """
+        pass
+
     def _set_parameter(self):
         self._logger.debug('run _set_parameter')
 
@@ -120,6 +126,15 @@ class DataSineLoad():
         encoding = 'utf-8'
         self.data = pd.read_csv(self.data, encoding=encoding)
 
+    def _meta_to_float(self):
+
+        for key, d in self.metadata.items():
+            try:
+                f = float(d.replace(',', '.'))
+                self.metadata[key] = f
+            except:
+                pass
+
     def _standardize_data(self):
         self._logger.debug('run _standardize_data')
 
@@ -143,11 +158,23 @@ class DataSineLoad():
 
                     break
 
+    def _modify_meta(self):
+        pass
+
     def _validate_data(self):
         self._logger.debug('run _validate_data')
 
         for name in self.val_col_names:
             if not name in self.data.columns:
+
+                # check if value in metadata:
+                if name in self.metadata.keys():
+
+                    self.data[name] = self.metadata[name]
+
+                else:
+
+                    print(name)
                     raise
 
     def _validate_meta(self):
@@ -157,6 +184,17 @@ class DataSineLoad():
             if not name in self.metadata:
                 raise
 
+    def _post_string_to_float(self):
+
+        sel = self.data.select_dtypes(include=['object'])
+
+        if sel.empty:
+            return
+
+        for col in sel.columns:
+            self.data[col] = pd.to_numeric(self.data[col].str.replace(
+                ',', '.'))
+
     def _post_apply_units(self):
 
         for col in ['s_hor_sum', 's_hor_1', 's_hor_2']:
@@ -209,6 +247,13 @@ class DataSineLoad():
 
         for idx, d in data_gp:
 
+            if d.empty: continue
+
+            if any(d['f'] <= 0.0): continue
+
+            #reset N
+            d['N'] = d['N'] - d['N'].iloc[0] + 1
+
             idx_diff = np.diff(d.index)
             dt_mean = idx_diff.mean()
 
@@ -249,129 +294,27 @@ class DataSineLoad():
         (a): Based on window of TP-Asphalt
         (b) last N cycles
         
+        DUMMY FUNCTION
         """
-
+        pass
-        self._logger.debug('run _fit_select_data')
-
-        def sel_df(df, num=5):
-
-            N = df['N'].unique()
-            freq = float(df['f'].unique()[0])
-
-            # define cycles to select
-            if freq == 10.0:
-                Nfrom = 98
-                Nto = 103
-            elif freq == 5.0:
-                Nfrom = 93
-                Nto = 97
-            elif freq == 3.0:
-                Nfrom = 43
-                Nto = 47
-            elif freq == 1.0:
-                Nfrom = 13
-                Nto = 17
-            elif freq == 0.3:
-                Nfrom = 8
-                Nto = 12
-            elif freq == 0.1:
-                Nfrom = 3
-                Nto = 7
-            else:
-                Nfrom = None
-                Nto = None
-
-            # Fall 1: nicht alle LW in Datei
-            if (max(N) < Nto) & (len(N) >= num):
-                df_sel = df[(df['N'] >= N[-num]) & (df['N'] <= N[-1])]
-
-            # Fall 2:
-            else:
-
-                if Nfrom != None:
-                    if len(N) > Nto - Nfrom:
-                        df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)]
-
-            return df_sel
-
-        if not isinstance(self.data, list):
-            if self.number_of_load_cycles_for_analysis > 1:
-                df_sel = [
-                    sel_df(self.data,
-                           num=self.number_of_load_cycles_for_analysis)
-                ]
-            else:
-                df_sel = [self.data]
-
-        else:
-            df_sel = []
-            for d in self.data:
-                if self.number_of_load_cycles_for_analysis > 1:
-                    d_sel = sel_df(d,
-                                   num=self.number_of_load_cycles_for_analysis)
-                else:
-                    d_sel = d
-
-                df_sel.append(d_sel)
-
-        # replace data
-        self.data = df_sel
 
     def _calc(self):
+        """
+        Calculate Results        
+        DUMMY FUNCTION
+        """
 
-        self.fit = []
+        self._logger.info('run _calc base')
-        for idx_data, data in enumerate(self.data):
+        print('run BASE')
 
-            if data is None: continue
+    def save(self):
-            if len(data) < 10: continue
+        '''
+        save results to database
         
-            data.index = data.index - data.index[0]
+        DUMMY FUNCTION
+        '''
 
-            res_temp = {}
+        pass
-
-            x = data.index.values
-
-            freq = np.round(float(data['f'].unique()), 2)
-            sigma = float(data['sigma'].unique())
-            temperature = float(data['T'].unique())
-
-            for idxcol, col in enumerate(self.columns_analyse):
-
-                if not col in data.columns: continue
-                y = data[col].values
-
-                res = fit_cos(x, y, freq=freq)
-
-                for key, value in res.items():
-                    res_temp[f'fit_{col}_{key}'] = value
-
-                res_temp[f'fit_{col}_max'] = max(y)
-                res_temp[f'fit_{col}_min'] = min(y)
-
-                res_temp['f'] = freq
-                res_temp['sigma'] = sigma
-                res_temp['T'] = temperature
-
-            ## Stiffness
-            deltaF = res_temp['fit_F_amp']
-            nu = calc_nu(temperature)
-            res_temp['nu'] = nu
-
-            h = float(self.metadata['speciment_height'])
-
-            deltaU = res_temp['fit_s_hor_sum_amp']
-
-            res_temp['E'] = (deltaF * (0.274 + nu)) / (h * deltaU)
-
-            self.fit.append(res_temp)
-
-        self.fit = pd.DataFrame.from_records(self.fit)
-
-        self.fit = self.fit.set_index(['T', 'f', 'sigma'])
-
-        nsamples = len(self.fit)
-        self._logger.info(f'fitting finished, add {nsamples} samples')
-        self._logger.debug(self.fit['E'])
 
     def _pre_run(self):
 
@@ -379,6 +322,7 @@ class DataSineLoad():
             self._read_from_s3_to_bytesio()
 
         self._calc_hash_of_bytesio()
+        self._which_machine()
         self._set_parameter()
         self.update_parameter()
         self._define_units()
@@ -387,12 +331,15 @@ class DataSineLoad():
         self._logger.info('run task')
 
         self._process_data()
+        self._meta_to_float()
 
         self._standardize_data()
         self._standardize_meta()
+        self._modify_meta()
         self._validate_data()
         self._validate_meta()
 
+        self._post_string_to_float()
         self._post_select_importent_columns()
         self._post_apply_units()
         self._post_calc_missiong_values()

src/paveit/labtest/citt.py

@@ -4,101 +4,374 @@ from csv import reader
 
 import numpy as np
 import pandas as pd
+from bson import ObjectId
+from paveit import calc_nu, fit_cos
+from paveit.datamodels import CITTSiffness, CITTSiffnessResults
+from paveit.io import read_geosys
 from paveit.labtest import DataSineLoad
 
 
 class CITTBase(DataSineLoad):
 
-    def _calc(self):
+    def _sel_df(self, df, num=5, shift=-1):
-        return (self.df.mean().mean(), self.df.max().max())
 
+        N = df['N'].unique()
+        n_N = len(N)
+        max_N = max(N)
+        min_N = min(N)
 
-class CITT_KIT(DataSineLoad):
+        freq = float(df['f'].unique()[0])
+
+        # define cycles to select
+        if freq == 10.0:
+            Nfrom = 98
+            Nto = 103
+        elif freq == 5.0:
+            Nfrom = 93
+            Nto = 97
+        elif freq == 3.0:
+            Nfrom = 43
+            Nto = 47
+        elif freq == 1.0:
+            Nfrom = 13
+            Nto = 17
+        elif freq == 0.3:
+            Nfrom = 8
+            Nto = 12
+        elif freq == 0.1:
+            Nfrom = 3
+            Nto = 7
+        else:
+            Nfrom = None
+            Nto = None
+
+        # Fall 1: nur num Lastwechsel
+        if n_N < num:
+            df_sel = None
+        elif n_N == num:
+            df_sel = df
+        # Fall 2: nicht alle LW in Datei
+        elif (max_N < Nto) & (n_N > num):
+
+            df_sel = df[(df['N'] >= N[-num + shift])
+                        & (df['N'] <= N[-1 + shift])]
+
+        # Fall 3: Auswahl wie oben definiert
+        elif (Nfrom >= min_N) & (Nto < max_N):
+            df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)]
+        # Fall 4: Auswahl unbekannt
+        else:
+            df_sel = None
+
+        return df_sel
+
+    def _fit_select_data(self):
+        """ 
+        select N load cycles from original data 
+        (a): Based on window of TP-Asphalt
+        (b) last N cycles
+        
+        """
+
+        self._logger.debug('run _fit_select_data')
+
+        self.max_N_in_data = []
+
+        if not isinstance(self.data, list):
+            if self.number_of_load_cycles_for_analysis > 1:
+
+                self.max_N_in_data.append(self.data['N'].max())
+
+                df_sel = [
+                    self._sel_df(self.data,
+                                 num=self.number_of_load_cycles_for_analysis)
+                ]
+            else:
+                df_sel = [self.data]
+
+        else:
+            df_sel = []
+            for d in self.data:
+
+                self.max_N_in_data.append(d['N'].max())
+
+                if self.number_of_load_cycles_for_analysis > 1:
+                    d_sel = self._sel_df(
+                        d, num=self.number_of_load_cycles_for_analysis)
+                else:
+                    d_sel = d
+
+                df_sel.append(d_sel)
+
+        # replace data
+        self.data = df_sel
 
     def _calc(self):
-        return (self.df.mean().mean(), self.df.max().max())
+
+        self._logger.info('run _calc CITT')
+        print('run CITT')
+
+        self.fit = []
+        for idx_data, data in enumerate(self.data):
+
+            if data is None: continue
+            if len(data) < 10: continue
+
+            try:
+
+                data.index = data.index - data.index[0]
+
+                res_temp = {}
+                res_temp['idx'] = idx_data
+
+                x = data.index.values
+
+                freq = np.round(float(data['f'].unique()), 2)
+                sigma = float(data['sigma'].unique())
+                temperature = float(data['T'].unique())
+
+                for idxcol, col in enumerate(self.columns_analyse):
+
+                    if not col in data.columns: continue
+                    y = data[col].values
+
+                    res = fit_cos(x, y, freq=freq)
+
+                    for key, value in res.items():
+                        res_temp[f'fit_{col}_{key}'] = value
+
+                    res_temp[f'fit_{col}_max'] = max(y)
+                    res_temp[f'fit_{col}_min'] = min(y)
+
+                # add more metadata
+                res_temp['f_set'] = freq
+                res_temp['sigma_set'] = sigma
+                res_temp['T_set'] = temperature
+
+                res_temp['N_from'] = data['N'].min()
+                res_temp['N_to'] = data['N'].max()
+                res_temp['N_tot'] = self.max_N_in_data[idx_data]
+
+                res_temp['n_samples_per_cycle'] = int(
+                    len(data) / (res_temp['N_to'] - res_temp['N_from'] + 1))
+
+                ## Stiffness
+                deltaF = res_temp['fit_F_amp']
+                nu = calc_nu(temperature)
+                res_temp['nu'] = nu
+
+                h = float(self.metadata['speciment_height'])
+
+                deltaU = res_temp['fit_s_hor_sum_amp']
+
+                res_temp['stiffness'] = (deltaF * (0.274 + nu)) / (h * deltaU)
+
+                # TODO: Überarbeiten und erweitern (ISSUE #2)
+                res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[
+                    'fit_s_hor_sum_phase']
+            except:
+                res_temp = None
+
+            self.fit.append(res_temp)
+
+        self.fit = pd.DataFrame.from_records(self.fit)
+
+        self.fit = self.fit.reset_index(drop=True).set_index('idx')
+
+        #self.fit = self.fit.set_index(['T', 'f', 'sigma'])
+
+        nsamples = len(self.fit)
+        self._logger.info(f'fitting finished, add {nsamples} samples')
+        self._logger.debug(self.fit['stiffness'])
+
+    def save(self,
+             org_id: ObjectId,
+             project_id: ObjectId,
+             material_id: ObjectId,
+             user_id: ObjectId,
+             meta: dict = {},
+             wp_id: ObjectId | None = None):
+        """
+        save results to mongodb
+        """
+
+        if not hasattr(self, 'fit'):
+            raise
+
+        # precheck data and results
+        assert len(self.data) == len(self.fit)
+
+        for idx_fit, fit in self.fit.iterrows():
+            data = self.data[idx_fit]
+
+            meta['filehash'] = self.filehash
+            meta['org_id'] = org_id
+            meta['project_id'] = project_id
+            meta['material'] = material_id
+            meta['user_id'] = user_id
+
+            #check if result in db
+            #n = CITTSiffness.objects(**meta).count()
+            #print(n)
+
+            # write data
+            data_dict = fit.to_dict()
+            data_dict.update(meta)
+
+            # remove 'fit_' from keys:
+            for key in list(data_dict.keys()):
+                if key.startswith('fit_'):
+                    data_dict[key[4:]] = data_dict[key]
+                    data_dict.pop(key)
+
+            # rename fields
+
+            def rename_field(d, old, new):
+                d[new] = d[old]
+                d.pop(old)
+
+            f = CITTSiffnessResults(**data_dict).save()
+
+            # required data
+            data_out = dict(
+                time=data.index,
+                F=list(data['F']),
+                N=list(data['N']),
+                s_hor_1=list(data['s_hor_1']),
+                s_hor_2=list(data['s_hor_2']),
+                s_hor_sum=list(data['s_hor_sum']),
+            )
+
+            # add optional datas
+            for col in ['s_piston']:
+                if col in data.columns:
+                    data_out[col] = list(data[col])
+
+            g = CITTSiffness(result=f.id, **data_out).save()
+
+
+class CITT_TUDresden(CITTBase):
+
+    def _which_machine(self):
+        """
+        check the file and try to get the machine from the data
+        """
+
+        self._machine = 'TIRA'
+
+    def _define_units(self):
+
+        if self._machine == 'TIRA':
+            self.unit_s = 1  #mm
+            self.unit_F = 1.  #N
+            self.unit_t = 1.  #s
+
+    def update_parameter(self):
+
+        if self._machine == 'TIRA':
+
+            self.meta_names_of_parameter = {
+                'sigma': ['Oberspannung'],
+                'f': ['Sollfrequnz'],
+                'T': ['Solltemperatur'],
+                #'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'],
+                #'Nto': ['Letzer Aufzeichnungslastwechsel', 'Last Cycle'],
+                't': ['Zeit'],
+                'speciment_diameter': ['PK-Durchmesser'],
+                'speciment_height': ['PK-Höhe'],
+            }  #list of names
+
+            self.data_column_names = {
+                'time': ['Zeit'],
+                'F': ['Kraft'],
+                'N': ['Lastwechsel'],
+                's_hor_1': ['IWA_1 2 mm'],
+                's_hor_2': ['IWA_2 2 mm'],
+                's_piston': ['Kolbenweg'],
+            }
 
     def _process_data(self):
-        self._logger.debug('convert bytes to pandas.DataFrame')
+
+        # -----------------------------------------------------------------------------------------------
+        # TIRA
+        # -----------------------------------------------------------------------------------------------
+        if self._machine == 'TIRA':
+            encoding = 'latin-1'
+            skiprows = 29
+            hasunits = True
+            splitsign = ':;'
+
+            # metadata from file
+            meta = {}
 
             self.data.seek(0)
-        with io.TextIOWrapper(self.data, encoding='latin-1') as read_obj:
+            f = self.data.readlines()
-            csv_reader = reader(read_obj, delimiter=';')
+            count = 0
 
-            read = False
+            for line in f:
+                count += 1
 
-            data = []
+                line = line.decode(encoding)
-            temp = []
 
-            for idx_row, row in enumerate(csv_reader):
+                #remove whitespace
-                if row == ['*****']:
+                linesplit = line.strip()
+                linesplit = linesplit.split(splitsign)
 
-                    if read == False:
+                if len(linesplit) == 2:
-                        read = True
-                    else:
-                        read = False
-                        data.append(temp)
 
-                        temp = []
+                    key = linesplit[0].strip()
+                    value = linesplit[1].split(';')[0].strip()
 
-                    continue
+                    meta[key] = value
 
-                if read:
+                if count >= skiprows:
+                    break
 
-                    row = [r.replace(',', '.') for r in row]
+            # data
+            self.data.seek(0)
+            data = pd.read_csv(self.data,
+                               encoding=encoding,
+                               header=None,
+                               skiprows=skiprows,
+                               decimal=',',
+                               thousands='.',
+                               sep=';')
 
-                    temp.append(row)
+            data = data.iloc[2:]
 
-        #convert to pandas
+            ## add header to df
+            self.data.seek(0)
+            f = self.data.readlines()
+            count = 0
 
-        res = []
+            for line in f:
 
-        freqs = [10.0, 5.0, 1.0, 0.1, 10.0]
+                line = line.decode(encoding)
 
-        for idx_data, d in enumerate(data):
+                count += 1
 
-            t = pd.DataFrame(d[3:])
+                if count >= skiprows:
-            t.columns = d[1]
+                    break
 
-            freq = freqs[idx_data]
+            #clean data
-            t['f'] = freq
+            data = data.dropna(axis=1)
 
-            for col in t.columns:
+            # add header from file
-                t[col] = pd.to_numeric(t[col])
+            head = line.split(';')
 
-            # add cycle number
+            data.columns = head
-            dt = 1. / freq
+            data.columns = [l.strip() for l in data.columns]
-
-            Nmax = int(np.ceil(t['ZEIT'].max() / dt))
-
-            N = np.zeros_like(t['ZEIT'])
-
-            for i in range(Nmax):
-                if i == 0:
-                    tmin = 0
-                    tmax = dt
-                else:
-                    tmax = (i + 1) * dt
-                    tmin = (i) * dt
-
-                idx = t[(t['ZEIT'] >= tmin) & (t['ZEIT'] < tmax)].index
-                N[idx] = i
-
-            t['N'] = N
-
-            res.append(t)
-
-        #remove second 10 Hz
-        res = pd.concat(res[:-1])
-
-        res['T'] = self.temperature
-        #res = res.sort_values(['f', 'ZEIT'])
 
         #define in class
-        self.data = res.reset_index()
+        self.data = data
+        self.metadata.update(meta)
+
+        # log infos
+        self._logger.info(self.metadata)
+        self._logger.info(self.data.head())
 
 
-class CITT_PTMDortmund(DataSineLoad):
+class CITT_PTMDortmund(CITTBase):
 
     def _define_units(self):
 
@@ -212,10 +485,7 @@ class CITT_PTMDortmund(DataSineLoad):
                     time_idx = None
             assert time_idx is not None
 
-            temp['N'] = 0
+            temp['N'] = 1
-
-            #BUG: Ist in Messdatei falsch definiert und wird von PTM angepasst.            '''
-            #for cycle in range(Nfrom, Nto+1):
 
             dt = 1.0 / frequency_test
 
@@ -230,8 +500,8 @@ class CITT_PTMDortmund(DataSineLoad):
                 #filter data
                 idx = temp[(time_idx >= tmin) & (time_idx < tmax)].index
 
-                #set cycle number
+                #set cycle number, cycle starts with 1
-                temp.loc[idx, 'N'] = cycle
+                temp.loc[idx, 'N'] = cycle + 1
 
                 cycle += 1
 
@@ -276,3 +546,201 @@ class CITT_PTMDortmund(DataSineLoad):
         # log infos
         self._logger.info(self.metadata)
         self._logger.info(self.data.head())
+
+
+class CITT_UniSiegen(CITTBase):
+
+    def _define_units(self):
+
+        self.unit_s = 1 / 1000.  #mm
+        self.unit_F = 1.0  #N
+        self.unit_t = 1.  #s
+
+    def update_parameter(self):
+
+        self.meta_names_of_parameter = {
+            'sigma': ['Oberspannung'],
+            'f': ['Frequenz'],
+            'T': ['Prüftemperatur'],
+            #'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'],
+            'Nto': ['Maximale Zyklenanzahl'],
+            't': ['Zeit'],
+        }  #list of names
+
+        self.data_column_names = {
+            'time': ['Zeit'],
+            'F': ['Kraft'],
+            's_hor_1': ['Radialweg vorn'],
+            's_hor_2': ['Radialweg hinten'],
+            's_piston': ['Kolbenweg'],
+            'N': ['Zyklenzähler'],
+        }
+
+    def _process_data(self):
+
+        meta, data = read_geosys(self.data, '015', metadata_ids=['003'])
+
+        #define in class
+        self.data = data.reset_index()
+        self.metadata.update(meta)
+
+        # log infos
+        self._logger.info(self.metadata)
+        self._logger.info(self.data.head())
+
+
+class CITT_BASt(CITTBase):
+
+    def _define_units(self):
+
+        self.unit_s = 1 / 1000.  #mm
+        self.unit_F = 1.0  #N
+        self.unit_t = 1.  #s
+
+    def update_parameter(self):
+
+        self.meta_names_of_parameter = {
+            'sigma': ['Oberspannung'],
+            'f': ['Versuchsart'],
+            'T': ['Prüftemperatur\r\n'],
+            #'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'],
+            'Nto': ['Maximale Zyklenanzahl'],
+            't': ['Zeit'],
+        }  #list of names
+
+        self.data_column_names = {
+            'time': ['Zeit'],
+            'F': ['Vertikalkraft'],
+            's_hor_1': ['Horizontalweg IWA vorn'],
+            's_hor_2': ['Horizontalweg IWA hinten'],
+            's_piston': ['Kolbenposition'],
+            'N': ['Zyklenzähler'],
+        }
+
+    def _process_data(self):
+
+        meta, data = read_geosys(self.data,
+                                 '047',
+                                 metadata_ids=['003', '005', '015'])
+
+        #define in class
+        self.data = data.reset_index()
+        self.metadata.update(meta)
+
+        # log infos
+        self._logger.info(self.metadata)
+        self._logger.info(self.data.head())
+
+    def _modify_meta(self):
+
+        s = self.metadata['f']
+        s = s.split('Hz')[0].split('Steifigkeit')[-1].strip().replace(',', '.')
+
+        self.metadata['f'] = float(s)
+
+        s = self.metadata['T']
+        s = s.split('°C')[0].strip().replace(',', '.')
+
+        self.metadata['T'] = float(s)
+
+
+class CITT_LaborHart(CITTBase):
+
+    def _define_units(self):
+
+        self.unit_s = 1.0  #mm
+        self.unit_F = 1.0  #N
+        self.unit_t = 1. / 1000.0  #s
+
+    def update_parameter(self):
+
+        self.meta_names_of_parameter = {
+            'sigma': ['Oberspannung'],
+            'T': ['Solltemperatur'],
+            't': ['TIME'],
+            'speciment_diameter': ['Probendurchmesser'],
+            'speciment_height': ['Probenhöhe'],
+        }  #list of names
+
+        self.data_column_names = {
+            'time': ['TIME'],
+            'f': ['FREQUENZ'],
+            'F': ['Load'],
+            's_hor_1': ['SENSOR 4'],
+            's_hor_2': ['SENSOR Extension'],
+            's_piston': ['Position'],
+            'N': ['Impulsnummer'],
+        }
+
+    def _process_data(self):
+
+        meta = {}
+
+        splitsign = ':;'
+        encoding = 'latin-1'
+        skiprows = 14
+
+        self.data.seek(0)
+        f = self.data.readlines()
+
+        count = 0
+
+        for line in f:
+            count += 1
+
+            #remove whitespace
+            line = line.decode(encoding)
+            linesplit = line.strip()
+            linesplit = linesplit.split(splitsign)
+
+            if len(linesplit) == 2:
+
+                meta[linesplit[0]] = linesplit[1]
+
+            if count >= skiprows:
+                break
+
+        # data
+        self.data.seek(0)
+
+        data = pd.read_csv(self.data,
+                           encoding=encoding,
+                           skiprows=skiprows,
+                           decimal=',',
+                           sep=';')
+
+        ## add header to df
+        self.data.seek(0)
+        f = self.data.readlines()
+        count = 0
+
+        for line in f:
+            count += 1
+
+            if count >= skiprows:
+                break
+
+        line = line.decode(encoding)
+        head = line.split(';')
+        data.columns = head
+
+        # FIX: Sigma nicht in Metadaten oder Messdaten enthalten
+        sigma = float(
+            os.path.split(self.filename)[-1].split('MPa')[0].strip().replace(
+                ',', '.'))
+
+        meta['sigma'] = sigma
+
+        #clean data
+        data = data.dropna(axis=1)
+
+        #remove whitespace
+        data.columns = [c.strip() for c in data.columns]
+
+        #define in class
+        self.data = data
+        self.metadata.update(meta)
+
+        # log infos
+        self._logger.info(self.metadata)
+        self._logger.info(self.data.head())