From 1bbb560f316baa7d79255b5eb1e858e31126cd36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Clau=C3=9F?= Date: Thu, 2 Mar 2023 17:31:39 +0100 Subject: [PATCH] =?UTF-8?q?Datenmodelle=20angepasst,=20einige=20Firmen=20i?= =?UTF-8?q?n=20CITT=20=C3=BCbernommen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/paveit/datamodels/__init__.py | 7 + src/paveit/datamodels/citt.py | 102 ++++ src/paveit/datamodels/data.py | 57 +++ src/paveit/datamodels/material.py | 89 ++++ src/paveit/datamodels/project.py | 42 ++ src/paveit/datamodels/sheartest.py | 107 +++++ src/paveit/datamodels/usermanagement.py | 51 ++ src/paveit/datamodels/workpackage.py | 33 ++ src/paveit/helper/__init__.py | 10 +- src/paveit/helper/mongo.py | 97 ++++ src/paveit/io/__init__.py | 3 + src/paveit/io/geosys.py | 227 +++++++++ src/paveit/labtest/base.py | 183 +++---- src/paveit/labtest/citt.py | 602 +++++++++++++++++++++--- 14 files changed, 1421 insertions(+), 189 deletions(-) create mode 100644 src/paveit/datamodels/__init__.py create mode 100644 src/paveit/datamodels/citt.py create mode 100644 src/paveit/datamodels/data.py create mode 100644 src/paveit/datamodels/material.py create mode 100644 src/paveit/datamodels/project.py create mode 100644 src/paveit/datamodels/sheartest.py create mode 100644 src/paveit/datamodels/usermanagement.py create mode 100644 src/paveit/datamodels/workpackage.py create mode 100644 src/paveit/helper/mongo.py create mode 100644 src/paveit/io/__init__.py create mode 100644 src/paveit/io/geosys.py diff --git a/src/paveit/datamodels/__init__.py b/src/paveit/datamodels/__init__.py new file mode 100644 index 0000000..0f62da9 --- /dev/null +++ b/src/paveit/datamodels/__init__.py @@ -0,0 +1,7 @@ +from .citt import * +from .data import * +from .material import * +from .project import * +from .sheartest import * +from .usermanagement import * +from .workpackage import * diff --git a/src/paveit/datamodels/citt.py b/src/paveit/datamodels/citt.py new file mode 100644 index 0000000..1c31186 --- /dev/null +++ b/src/paveit/datamodels/citt.py @@ -0,0 +1,102 @@ +import datetime + +from mongoengine import * + +from .material import Material +from .project import Project +from .usermanagement import Organisation, User +from .workpackage import Workpackage + + +class CyclicIndirectTensileTest(Document): + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + standard = StringField(default='TP Asphalt Teil 24') + + org_id = LazyReferenceField(Organisation, required=True) + project_id = LazyReferenceField(Project, required=True) + workpackage_id = LazyReferenceField(Workpackage, required=False) + + user_id = LazyReferenceField(User, + required=True, + reverse_delete_rule=DO_NOTHING) + + material = LazyReferenceField(Material, required=True) + + tags = ListField(StringField()) + + machine = StringField(default=None) + + filehash = StringField(required=True) + speciment_name = StringField() + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'citt', + "db_alias": 'dblabtests', + } + + +class CITTSiffnessResults(CyclicIndirectTensileTest): + + #metadata + f_set = FloatField() + sigma_set = FloatField() + T_set = FloatField() + + N_from = IntField() + N_to = IntField() + N_tot = IntField() + n_samples_per_cycle = IntField() + + #results + stiffness = FloatField() + nu = FloatField() + phase = FloatField() + #required parameter + ## F + F_amp = FloatField() + F_freq = FloatField() + F_phase = FloatField() + F_offset = FloatField() + F_slope = FloatField() + F_r2 = FloatField() + F_min = FloatField() + F_max = FloatField() + ## S1 + s_hor_1_amp = FloatField() + s_hor_1_freq = FloatField() + s_hor_1_phase = FloatField() + s_hor_1_offset = FloatField() + s_hor_1_slope = FloatField() + s_hor_1_r2 = FloatField() + s_hor_1_min = FloatField() + s_hor_1_max = FloatField() + ## S2 + s_hor_2_amp = FloatField() + s_hor_2_freq = FloatField() + s_hor_2_phase = FloatField() + s_hor_2_offset = FloatField() + s_hor_2_slope = FloatField() + s_hor_2_r2 = FloatField() + s_hor_2_min = FloatField() + s_hor_2_max = FloatField() + ## S-Sum + s_hor_sum_amp = FloatField() + s_hor_sum_freq = FloatField() + s_hor_sum_phase = FloatField() + s_hor_sum_offset = FloatField() + s_hor_sum_slope = FloatField() + s_hor_sum_r2 = FloatField() + s_hor_sum_min = FloatField() + s_hor_sum_max = FloatField() + + #optional parameter \ No newline at end of file diff --git a/src/paveit/datamodels/data.py b/src/paveit/datamodels/data.py new file mode 100644 index 0000000..65a42b6 --- /dev/null +++ b/src/paveit/datamodels/data.py @@ -0,0 +1,57 @@ +import datetime + +from mongoengine import * + +from .citt import CyclicIndirectTensileTest +from .sheartest import DynamicShearTest + + +class RawData(Document): + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'rawdata', + "db_alias": 'dblabtests', + } + + +class DataSheartest(RawData): + + #results + result_id = LazyReferenceField(DynamicShearTest, + required=True, + reverse_delete_rule=CASCADE) + + # data + time = ListField(FloatField()) + F = ListField(FloatField()) + N = ListField(IntField()) + s_vert_1 = ListField(FloatField()) + s_vert_2 = ListField(FloatField()) + s_vert_sum = ListField(FloatField(), required=False) + s_piston = ListField(FloatField(), required=False) + + +class CITTSiffness(RawData): + + result = LazyReferenceField(CyclicIndirectTensileTest, + required=True, + reverse_delete_rule=CASCADE) + + # data + time = ListField(FloatField()) + F = ListField(FloatField()) + N = ListField(IntField()) + s_hor_1 = ListField(FloatField()) + s_hor_2 = ListField(FloatField()) + s_hor_sum = ListField(FloatField()) + s_piston = ListField(FloatField(), required=False) diff --git a/src/paveit/datamodels/material.py b/src/paveit/datamodels/material.py new file mode 100644 index 0000000..98c75df --- /dev/null +++ b/src/paveit/datamodels/material.py @@ -0,0 +1,89 @@ +import datetime +from re import T + +from mongoengine import * + +from .project import Project +from .usermanagement import Organisation, User + + +class Material(Document): + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + org_id = LazyReferenceField(Organisation, + required=True, + reverse_delete_rule=CASCADE) + + project_id = ListField(LazyReferenceField(Project, + required=False, + reverse_delete_rule=CASCADE), + required=True) + + user_id = LazyReferenceField(User, + required=False, + reverse_delete_rule=DO_NOTHING) + + tags = ListField(StringField()) + norm = StringField(required=True, default='TP Asphalt Teil 24') + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'materials', + 'indexes': [ + [("material", 1)], + [("name", 1)], + ] + } + + +class Asphalt(Material): + + name = StringField() + material = StringField() + bitumen = StringField(required=False) + + young_modulus = DictField() + fatigue = DictField() + + +class Bitumen(Material): + + name = StringField() + material = StringField() + + young_modulus = DictField() + + +class Bitumenemulsion(Material): + + name = StringField() + material = StringField() + + young_modulus = DictField() + + +class Epoxy(Material): + name = StringField() + material = StringField() + + young_modulus = DictField() + + +class Kompaktasphalt(Material): + name = StringField() + + +class Dummy(Material): + + name = StringField() + material = StringField() + + young_modulus = DictField() diff --git a/src/paveit/datamodels/project.py b/src/paveit/datamodels/project.py new file mode 100644 index 0000000..fe82cc2 --- /dev/null +++ b/src/paveit/datamodels/project.py @@ -0,0 +1,42 @@ +import datetime + +from mongoengine import * + +from .usermanagement import Organisation, User + + +class Project(Document): + + name = StringField(required=True) + name_short = StringField(required=False) + project_id = StringField(required=True) + client = StringField(required=False) + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + org_id = LazyReferenceField(Organisation, + required=True, + reverse_delete_rule=CASCADE) + + user_id = LazyReferenceField(User, + required=False, + reverse_delete_rule=DO_NOTHING) + + tags = ListField(StringField()) + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'projects', + 'indexes': [ + [("name_short", 1)], + [("name", 1)], + [("project_id", 1)], + ] + } diff --git a/src/paveit/datamodels/sheartest.py b/src/paveit/datamodels/sheartest.py new file mode 100644 index 0000000..ea4b486 --- /dev/null +++ b/src/paveit/datamodels/sheartest.py @@ -0,0 +1,107 @@ +import datetime + +from mongoengine import * + +from .material import Material +from .project import Project +from .usermanagement import Organisation, User +from .workpackage import Workpackage + + +class DynamicShearTest(Document): + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + org_id = LazyReferenceField(Organisation, required=True) + project_id = LazyReferenceField(Project, required=True) + workpackage_id = LazyReferenceField(Workpackage, required=False) + + user_id = LazyReferenceField(User, + required=True, + reverse_delete_rule=DO_NOTHING) + + material1 = LazyReferenceField(Material, required=True) + material2 = LazyReferenceField(Material, required=True) + bounding = LazyReferenceField(Material, required=True) + + gap_width = FloatField(default=1.0) + + tags = ListField(StringField()) + + standard = StringField(default='TP Asphalt Teil 24') + + machine = StringField(default=None) + + filehash = StringField(required=True) + speciment_name = StringField() + + meta = { + 'allow_inheritance': + True, + 'index_opts': {}, + 'index_background': + True, + 'index_cls': + False, + 'auto_create_index': + True, + 'collection': + 'sheartest', + 'indexes': [ + [("lab", 1)], + [("speciment_name", 1)], + [("project", 1)], + [("bruch", 1)], + [("lab", 1), ("project", 1)], + [("lab", 1), ("project", 1), ("workpackage", 1)], + [("lab", 1), ("project", 1), ("bounding", 1)], + ] + } + + +class DynamicShearTestExtension(DynamicShearTest): + + #metadata + f = FloatField(required=True) + sigma_normal = FloatField(required=True) + T = FloatField(required=True) + extension = FloatField(required=True) + + stiffness = FloatField(required=True) + bruch = BooleanField(required=True) + #fit parameter + ## F + fit_amp_F = FloatField(required=True) + fit_freq_F = FloatField(required=True) + fit_phase_F = FloatField(required=True) + fit_offset_F = FloatField(required=True) + fit_slope_F = FloatField(required=True) + ## S1 + fit_amp_s_vert_1 = FloatField(required=True) + fit_freq_s_vert_1 = FloatField(required=True) + fit_phase_s_vert_1 = FloatField(required=True) + fit_offset_s_vert_1 = FloatField(required=True) + fit_slope_s_vert_1 = FloatField(required=True) + r2_s_vert_1 = FloatField(required=True) + ## S2 + fit_amp_s_vert_2 = FloatField(required=True) + fit_freq_s_vert_2 = FloatField(required=True) + fit_phase_s_vert_2 = FloatField(required=True) + fit_offset_s_vert_2 = FloatField(required=True) + fit_slope_s_vert_2 = FloatField(required=True) + r2_s_vert_2 = FloatField(required=True) + ## S-Sum + fit_amp_s_vert_sum = FloatField(required=True) + fit_freq_s_vert_sum = FloatField(required=True) + fit_phase_s_vert_sum = FloatField(required=True) + fit_offset_s_vert_sum = FloatField(required=True) + fit_slope_s_vert_sum = FloatField(required=True) + r2_s_vert_sum = FloatField(required=True) + ## r2 + r2_F = FloatField(required=True) + r2_s_vert_1 = FloatField(required=True) + r2_s_vert_2 = FloatField(required=True) + r2_s_vert_sum = FloatField(required=True) diff --git a/src/paveit/datamodels/usermanagement.py b/src/paveit/datamodels/usermanagement.py new file mode 100644 index 0000000..cbf8773 --- /dev/null +++ b/src/paveit/datamodels/usermanagement.py @@ -0,0 +1,51 @@ +import datetime + +from mongoengine import * + + +class Organisation(Document): + + name = StringField(required=True) + name_short = StringField(required=True) + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + labtest_citt = StringField(required=False) + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'organisation', + 'db_alias': 'dbusers', + } + + +class User(Document): + + active = BooleanField(required=True, default=True) + + org_id = LazyReferenceField(Organisation, + required=True, + reverse_delete_rule=CASCADE) + + date_added = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + name = StringField(required=True) + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'user' + } \ No newline at end of file diff --git a/src/paveit/datamodels/workpackage.py b/src/paveit/datamodels/workpackage.py new file mode 100644 index 0000000..16040b6 --- /dev/null +++ b/src/paveit/datamodels/workpackage.py @@ -0,0 +1,33 @@ +import datetime + +from mongoengine import * + +from .project import Project +from .usermanagement import User + + +class Workpackage(Document): + + name = StringField(required=True) + name_short = StringField(required=False) + wp_id = StringField(required=True) + + project_id = LazyReferenceField(Project, required=True) + + user_id = LazyReferenceField(User, + required=False, + reverse_delete_rule=DO_NOTHING) + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'workpackages' + } \ No newline at end of file diff --git a/src/paveit/helper/__init__.py b/src/paveit/helper/__init__.py index a53354b..c841f41 100644 --- a/src/paveit/helper/__init__.py +++ b/src/paveit/helper/__init__.py @@ -1,8 +1,10 @@ from .filehandling import read_file_to_bytesio from .filehasher import calc_hash_of_bytes from .minio import get_minio_client_archive, get_minio_client_processing +from .mongo import connect_mongo_dbs, mongo_get_results -__all__ = ['read_file_to_bytesio', - 'get_minio_client_archive', 'get_minio_client_processing', - 'calc_hash_of_bytes' - ] \ No newline at end of file +__all__ = [ + 'read_file_to_bytesio', 'connect_mongo_dbs', 'mongo_get_results', + 'get_minio_client_processing', 'get_minio_client_archive', + 'calc_hash_of_bytes' +] diff --git a/src/paveit/helper/mongo.py b/src/paveit/helper/mongo.py new file mode 100644 index 0000000..c61d425 --- /dev/null +++ b/src/paveit/helper/mongo.py @@ -0,0 +1,97 @@ +from bson import ObjectId +from mongoengine import connect as mongo_connect +from pandas import DataFrame + + +def connect_mongo_db_labtests(username='admin', + password='admin', + host='mongo', + authentication_source='admin'): + + mongo_connect('labtests', + username=username, + password=password, + host=host, + authentication_source=authentication_source, + alias='dblabtests') + + +def connect_mongo_db_usser(username='admin', + password='admin', + host='mongo', + authentication_source='admin'): + + mongo_connect('users', + username=username, + password=password, + host=host, + authentication_source=authentication_source, + alias='dbusers') + + +def connect_mongo_dbs(username='admin', + password='admin', + host='mongo', + authentication_source='admin'): + + connect_mongo_db_labtests(username=username, + password=password, + host=host, + authentication_source=authentication_source) + + connect_mongo_db_usser(username=username, + password=password, + host=host, + authentication_source=authentication_source) + + +def mongo_upload_results(resultsmodel, results: DataFrame, datamodel, + data: DataFrame, filehash: str, org_id: ObjectId, + project_id: ObjectId, material_id: ObjectId, + user_id: ObjectId): + + for idx, res in results.iterrows(): + + #upload results + meta['filehash'] = filehash + meta['org_id'] = org_id + meta['project_id'] = project_id + meta['material'] = material_id + meta['user_id'] = user_id + + #check if result in db + #n = CITTSiffness.objects(**meta).count() + #print(n) + + # write data + data_dict = res.to_dict() + data_dict.update(meta) + + f = resultsmodel(**data_dict).save() + + # upload data + data_sel = data[idx_fit] + + # required data + data_out = dict( + time=data_sel.index, + F=list(data_sel['F']), + N=list(data_sel['N']), + s_hor_1=list(data_sel['s_hor_1']), + s_hor_2=list(data_sel['s_hor_2']), + s_hor_sum=list(data_sel['s_hor_sum']), + ) + #optional data + for col in ['S_piston']: + if col in data_sel.columns: + data_out[col] = data_sel[col] + + g = datamodel(result=f.id, **data_out).save() + + +def mongo_get_results(resultsmodel, results: DataFrame, datamodel, + data: DataFrame, filehash: str, org_id: ObjectId, + project_id: ObjectId, material_id: ObjectId, + user_id: ObjectId): + + return True \ No newline at end of file diff --git a/src/paveit/io/__init__.py b/src/paveit/io/__init__.py new file mode 100644 index 0000000..b438175 --- /dev/null +++ b/src/paveit/io/__init__.py @@ -0,0 +1,3 @@ +from .geosys import read_geosys + +__all__ = ["read_geosys"] diff --git a/src/paveit/io/geosys.py b/src/paveit/io/geosys.py new file mode 100644 index 0000000..f980b40 --- /dev/null +++ b/src/paveit/io/geosys.py @@ -0,0 +1,227 @@ +import csv +import os +from io import BytesIO +from sys import getsizeof + +from numpy import array +from pandas import DataFrame + + +def detect_tabnum(filename, tabstr, encoding='utf-8'): + filename = os.path.normpath(filename) + + tabstr = tabstr.lower() + + #Einlesen + with open(filename, 'r', encoding=encoding) as inFile: + reader = csv.reader(inFile, delimiter='\t') + counter = 0 + for row in reader: + + row = [r.lower() for r in row] + if any(tabstr in mystring for mystring in row): + if 'plain' in row: + return row[1] + + counter += 1 + + if counter > 100: + return False + + +def str2float(str): + try: + str = str.replace(',', '.') + return float(str) + except: + return None + + +def read_geosys(buffer: BytesIO, + table, + pkdata='001', + metadata_ids=['003', '015'], + encoding='utf-8', + to_si=False, + debug=False): + ''' + + :param buffer: Bytes IO Object + :param table: Table-Number + :param pkdata: Table-Number of speciment definitions, default: 1 + :param encoding: Encoding, default: utf-8 + :param debug: debug-mode + :return: + + ''' + + try: + dictOut = {} + dictOut['durch'] = 0 + dictOut['hoehe'] = 0 + + #--------------------------------------------------------------------- + #Daten einlesen und umwandeln + #--------------------------------------------------------------------- + + data = [] + + #Einlesen + buffer.seek(0) + + lines_file = buffer.readlines() + + for line in lines_file: + try: + line = line.decode(encoding) + line = line.split('\t') + + if len(line) > 2: + data.append(line) + + except: + pass + + if debug: + print('Anz. Datensätze: ', str(len(data)), getsizeof(data)) + #aufräumen + ##Datenstruktur anlegen + + data_processed = {} + data_processed['head'] = [] + data_processed['metadata'] = {} + data_processed['data'] = [] + + for i in metadata_ids: + data_processed['metadata'][i] = [] + + for idx, d in enumerate(data): + try: + v = d[0][0:3] + if v in pkdata: data_processed['head'].append(d) + if v in metadata_ids: data_processed['metadata'][v].append(d) + if v in table: data_processed['data'].append(d) + + except: + pass + + # replace object + data = data_processed + + if debug: + print('data_clean fin') + + ## Header aufbereiten + + for idx, row in enumerate(data['head']): + if idx == 0: + id_durchmesser = None + id_hoehe = None + id_name = None + + for idx_name, name in enumerate(row): + name_lower = name.lower() + + if any(map(name_lower.__contains__, ['durchmesser'])): + id_durchmesser = idx_name + + elif any(map(name_lower.__contains__, ['bezeichnung'])): + id_name = idx_name + + elif any(map(name_lower.__contains__, ['höhe'])): + id_hoehe = idx_name + + if debug: + print(id_durchmesser, id_hoehe, id_name) + elif idx == 1: + unit_durch = None + unit_hoehe = None + + try: + unit_durch = row[id_durchmesser] + unit_hoehe = row[id_hoehe] + except: + pass + + elif idx == 2: + durchmesser = None + hoehe = None + name = None + try: + durchmesser = str2float(row[id_durchmesser]) + hoehe = str2float(row[id_hoehe]) + name = row[id_name] + + except: + pass + + header = { + 'speciment_diameter': durchmesser, + 'speciment_height': hoehe, + 'name': name, + 'unit_h': unit_hoehe, + 'unit_d': unit_durch + } + + meta = data['metadata'] + + for key in meta.keys(): + sel = meta[key] + + assert len(sel[0]) == len(sel[2]) + header_append = { + sel[0][i]: sel[2][i].strip() + for i in range(len(sel[0])) + } + + header.update(header_append) + + if debug: + print('header\n', header) + + # add metadata to header + + ## Daten in Pandas DataFrame umwandeln + if debug: + print('daten umwandel') + + temp = [] + for idx, row in enumerate(data['data']): + if idx == 0: + if debug: + print('head') + data_head = [] + for idx_name, name in enumerate(row): + if idx_name <= 1: continue + data_head.append(name) + elif idx == 1: + data_units = [] + for idx_name, name in enumerate(row): + if idx_name <= 1: continue + data_units.append(name) + else: + t = [] + for idx_col, value in enumerate(row): + if idx_col <= 1: + continue + else: + t.append(str2float(value)) + + temp.append(t) + + data = array(temp) + + if debug: + print(data_head, data_units) + + ## Bezeichnungen der Daten normalisieren + # Pandas DataFrame erstellen + data = DataFrame(data=data, columns=data_head) + if debug: + print(data.head()) + + return header, data + + except: + print('Fehler beim lesen') + raise diff --git a/src/paveit/labtest/base.py b/src/paveit/labtest/base.py index 0c4222a..81f4a3e 100644 --- a/src/paveit/labtest/base.py +++ b/src/paveit/labtest/base.py @@ -40,6 +40,12 @@ class DataSineLoad(): self._pre_run() + def _which_machine(self): + """ + check the file and try to get the machine from the data + """ + pass + def _set_parameter(self): self._logger.debug('run _set_parameter') @@ -120,6 +126,15 @@ class DataSineLoad(): encoding = 'utf-8' self.data = pd.read_csv(self.data, encoding=encoding) + def _meta_to_float(self): + + for key, d in self.metadata.items(): + try: + f = float(d.replace(',', '.')) + self.metadata[key] = f + except: + pass + def _standardize_data(self): self._logger.debug('run _standardize_data') @@ -143,12 +158,24 @@ class DataSineLoad(): break + def _modify_meta(self): + pass + def _validate_data(self): self._logger.debug('run _validate_data') for name in self.val_col_names: if not name in self.data.columns: - raise + + # check if value in metadata: + if name in self.metadata.keys(): + + self.data[name] = self.metadata[name] + + else: + + print(name) + raise def _validate_meta(self): self._logger.debug('run _validate_meta') @@ -157,6 +184,17 @@ class DataSineLoad(): if not name in self.metadata: raise + def _post_string_to_float(self): + + sel = self.data.select_dtypes(include=['object']) + + if sel.empty: + return + + for col in sel.columns: + self.data[col] = pd.to_numeric(self.data[col].str.replace( + ',', '.')) + def _post_apply_units(self): for col in ['s_hor_sum', 's_hor_1', 's_hor_2']: @@ -209,6 +247,13 @@ class DataSineLoad(): for idx, d in data_gp: + if d.empty: continue + + if any(d['f'] <= 0.0): continue + + #reset N + d['N'] = d['N'] - d['N'].iloc[0] + 1 + idx_diff = np.diff(d.index) dt_mean = idx_diff.mean() @@ -249,129 +294,27 @@ class DataSineLoad(): (a): Based on window of TP-Asphalt (b) last N cycles + DUMMY FUNCTION """ - - self._logger.debug('run _fit_select_data') - - def sel_df(df, num=5): - - N = df['N'].unique() - freq = float(df['f'].unique()[0]) - - # define cycles to select - if freq == 10.0: - Nfrom = 98 - Nto = 103 - elif freq == 5.0: - Nfrom = 93 - Nto = 97 - elif freq == 3.0: - Nfrom = 43 - Nto = 47 - elif freq == 1.0: - Nfrom = 13 - Nto = 17 - elif freq == 0.3: - Nfrom = 8 - Nto = 12 - elif freq == 0.1: - Nfrom = 3 - Nto = 7 - else: - Nfrom = None - Nto = None - - # Fall 1: nicht alle LW in Datei - if (max(N) < Nto) & (len(N) >= num): - df_sel = df[(df['N'] >= N[-num]) & (df['N'] <= N[-1])] - - # Fall 2: - else: - - if Nfrom != None: - if len(N) > Nto - Nfrom: - df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)] - - return df_sel - - if not isinstance(self.data, list): - if self.number_of_load_cycles_for_analysis > 1: - df_sel = [ - sel_df(self.data, - num=self.number_of_load_cycles_for_analysis) - ] - else: - df_sel = [self.data] - - else: - df_sel = [] - for d in self.data: - if self.number_of_load_cycles_for_analysis > 1: - d_sel = sel_df(d, - num=self.number_of_load_cycles_for_analysis) - else: - d_sel = d - - df_sel.append(d_sel) - - # replace data - self.data = df_sel + pass def _calc(self): + """ + Calculate Results + DUMMY FUNCTION + """ - self.fit = [] - for idx_data, data in enumerate(self.data): + self._logger.info('run _calc base') + print('run BASE') - if data is None: continue - if len(data) < 10: continue + def save(self): + ''' + save results to database + + DUMMY FUNCTION + ''' - data.index = data.index - data.index[0] - - res_temp = {} - - x = data.index.values - - freq = np.round(float(data['f'].unique()), 2) - sigma = float(data['sigma'].unique()) - temperature = float(data['T'].unique()) - - for idxcol, col in enumerate(self.columns_analyse): - - if not col in data.columns: continue - y = data[col].values - - res = fit_cos(x, y, freq=freq) - - for key, value in res.items(): - res_temp[f'fit_{col}_{key}'] = value - - res_temp[f'fit_{col}_max'] = max(y) - res_temp[f'fit_{col}_min'] = min(y) - - res_temp['f'] = freq - res_temp['sigma'] = sigma - res_temp['T'] = temperature - - ## Stiffness - deltaF = res_temp['fit_F_amp'] - nu = calc_nu(temperature) - res_temp['nu'] = nu - - h = float(self.metadata['speciment_height']) - - deltaU = res_temp['fit_s_hor_sum_amp'] - - res_temp['E'] = (deltaF * (0.274 + nu)) / (h * deltaU) - - self.fit.append(res_temp) - - self.fit = pd.DataFrame.from_records(self.fit) - - self.fit = self.fit.set_index(['T', 'f', 'sigma']) - - nsamples = len(self.fit) - self._logger.info(f'fitting finished, add {nsamples} samples') - self._logger.debug(self.fit['E']) + pass def _pre_run(self): @@ -379,6 +322,7 @@ class DataSineLoad(): self._read_from_s3_to_bytesio() self._calc_hash_of_bytesio() + self._which_machine() self._set_parameter() self.update_parameter() self._define_units() @@ -387,12 +331,15 @@ class DataSineLoad(): self._logger.info('run task') self._process_data() + self._meta_to_float() self._standardize_data() self._standardize_meta() + self._modify_meta() self._validate_data() self._validate_meta() + self._post_string_to_float() self._post_select_importent_columns() self._post_apply_units() self._post_calc_missiong_values() diff --git a/src/paveit/labtest/citt.py b/src/paveit/labtest/citt.py index 3931821..0318935 100644 --- a/src/paveit/labtest/citt.py +++ b/src/paveit/labtest/citt.py @@ -4,101 +4,374 @@ from csv import reader import numpy as np import pandas as pd +from bson import ObjectId +from paveit import calc_nu, fit_cos +from paveit.datamodels import CITTSiffness, CITTSiffnessResults +from paveit.io import read_geosys from paveit.labtest import DataSineLoad class CITTBase(DataSineLoad): - def _calc(self): - return (self.df.mean().mean(), self.df.max().max()) + def _sel_df(self, df, num=5, shift=-1): + N = df['N'].unique() + n_N = len(N) + max_N = max(N) + min_N = min(N) -class CITT_KIT(DataSineLoad): + freq = float(df['f'].unique()[0]) + + # define cycles to select + if freq == 10.0: + Nfrom = 98 + Nto = 103 + elif freq == 5.0: + Nfrom = 93 + Nto = 97 + elif freq == 3.0: + Nfrom = 43 + Nto = 47 + elif freq == 1.0: + Nfrom = 13 + Nto = 17 + elif freq == 0.3: + Nfrom = 8 + Nto = 12 + elif freq == 0.1: + Nfrom = 3 + Nto = 7 + else: + Nfrom = None + Nto = None + + # Fall 1: nur num Lastwechsel + if n_N < num: + df_sel = None + elif n_N == num: + df_sel = df + # Fall 2: nicht alle LW in Datei + elif (max_N < Nto) & (n_N > num): + + df_sel = df[(df['N'] >= N[-num + shift]) + & (df['N'] <= N[-1 + shift])] + + # Fall 3: Auswahl wie oben definiert + elif (Nfrom >= min_N) & (Nto < max_N): + df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)] + # Fall 4: Auswahl unbekannt + else: + df_sel = None + + return df_sel + + def _fit_select_data(self): + """ + select N load cycles from original data + (a): Based on window of TP-Asphalt + (b) last N cycles + + """ + + self._logger.debug('run _fit_select_data') + + self.max_N_in_data = [] + + if not isinstance(self.data, list): + if self.number_of_load_cycles_for_analysis > 1: + + self.max_N_in_data.append(self.data['N'].max()) + + df_sel = [ + self._sel_df(self.data, + num=self.number_of_load_cycles_for_analysis) + ] + else: + df_sel = [self.data] + + else: + df_sel = [] + for d in self.data: + + self.max_N_in_data.append(d['N'].max()) + + if self.number_of_load_cycles_for_analysis > 1: + d_sel = self._sel_df( + d, num=self.number_of_load_cycles_for_analysis) + else: + d_sel = d + + df_sel.append(d_sel) + + # replace data + self.data = df_sel def _calc(self): - return (self.df.mean().mean(), self.df.max().max()) + + self._logger.info('run _calc CITT') + print('run CITT') + + self.fit = [] + for idx_data, data in enumerate(self.data): + + if data is None: continue + if len(data) < 10: continue + + try: + + data.index = data.index - data.index[0] + + res_temp = {} + res_temp['idx'] = idx_data + + x = data.index.values + + freq = np.round(float(data['f'].unique()), 2) + sigma = float(data['sigma'].unique()) + temperature = float(data['T'].unique()) + + for idxcol, col in enumerate(self.columns_analyse): + + if not col in data.columns: continue + y = data[col].values + + res = fit_cos(x, y, freq=freq) + + for key, value in res.items(): + res_temp[f'fit_{col}_{key}'] = value + + res_temp[f'fit_{col}_max'] = max(y) + res_temp[f'fit_{col}_min'] = min(y) + + # add more metadata + res_temp['f_set'] = freq + res_temp['sigma_set'] = sigma + res_temp['T_set'] = temperature + + res_temp['N_from'] = data['N'].min() + res_temp['N_to'] = data['N'].max() + res_temp['N_tot'] = self.max_N_in_data[idx_data] + + res_temp['n_samples_per_cycle'] = int( + len(data) / (res_temp['N_to'] - res_temp['N_from'] + 1)) + + ## Stiffness + deltaF = res_temp['fit_F_amp'] + nu = calc_nu(temperature) + res_temp['nu'] = nu + + h = float(self.metadata['speciment_height']) + + deltaU = res_temp['fit_s_hor_sum_amp'] + + res_temp['stiffness'] = (deltaF * (0.274 + nu)) / (h * deltaU) + + # TODO: Überarbeiten und erweitern (ISSUE #2) + res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[ + 'fit_s_hor_sum_phase'] + except: + res_temp = None + + self.fit.append(res_temp) + + self.fit = pd.DataFrame.from_records(self.fit) + + self.fit = self.fit.reset_index(drop=True).set_index('idx') + + #self.fit = self.fit.set_index(['T', 'f', 'sigma']) + + nsamples = len(self.fit) + self._logger.info(f'fitting finished, add {nsamples} samples') + self._logger.debug(self.fit['stiffness']) + + def save(self, + org_id: ObjectId, + project_id: ObjectId, + material_id: ObjectId, + user_id: ObjectId, + meta: dict = {}, + wp_id: ObjectId | None = None): + """ + save results to mongodb + """ + + if not hasattr(self, 'fit'): + raise + + # precheck data and results + assert len(self.data) == len(self.fit) + + for idx_fit, fit in self.fit.iterrows(): + data = self.data[idx_fit] + + meta['filehash'] = self.filehash + meta['org_id'] = org_id + meta['project_id'] = project_id + meta['material'] = material_id + meta['user_id'] = user_id + + #check if result in db + #n = CITTSiffness.objects(**meta).count() + #print(n) + + # write data + data_dict = fit.to_dict() + data_dict.update(meta) + + # remove 'fit_' from keys: + for key in list(data_dict.keys()): + if key.startswith('fit_'): + data_dict[key[4:]] = data_dict[key] + data_dict.pop(key) + + # rename fields + + def rename_field(d, old, new): + d[new] = d[old] + d.pop(old) + + f = CITTSiffnessResults(**data_dict).save() + + # required data + data_out = dict( + time=data.index, + F=list(data['F']), + N=list(data['N']), + s_hor_1=list(data['s_hor_1']), + s_hor_2=list(data['s_hor_2']), + s_hor_sum=list(data['s_hor_sum']), + ) + + # add optional datas + for col in ['s_piston']: + if col in data.columns: + data_out[col] = list(data[col]) + + g = CITTSiffness(result=f.id, **data_out).save() + + +class CITT_TUDresden(CITTBase): + + def _which_machine(self): + """ + check the file and try to get the machine from the data + """ + + self._machine = 'TIRA' + + def _define_units(self): + + if self._machine == 'TIRA': + self.unit_s = 1 #mm + self.unit_F = 1. #N + self.unit_t = 1. #s + + def update_parameter(self): + + if self._machine == 'TIRA': + + self.meta_names_of_parameter = { + 'sigma': ['Oberspannung'], + 'f': ['Sollfrequnz'], + 'T': ['Solltemperatur'], + #'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'], + #'Nto': ['Letzer Aufzeichnungslastwechsel', 'Last Cycle'], + 't': ['Zeit'], + 'speciment_diameter': ['PK-Durchmesser'], + 'speciment_height': ['PK-Höhe'], + } #list of names + + self.data_column_names = { + 'time': ['Zeit'], + 'F': ['Kraft'], + 'N': ['Lastwechsel'], + 's_hor_1': ['IWA_1 2 mm'], + 's_hor_2': ['IWA_2 2 mm'], + 's_piston': ['Kolbenweg'], + } def _process_data(self): - self._logger.debug('convert bytes to pandas.DataFrame') - self.data.seek(0) - with io.TextIOWrapper(self.data, encoding='latin-1') as read_obj: - csv_reader = reader(read_obj, delimiter=';') + # ----------------------------------------------------------------------------------------------- + # TIRA + # ----------------------------------------------------------------------------------------------- + if self._machine == 'TIRA': + encoding = 'latin-1' + skiprows = 29 + hasunits = True + splitsign = ':;' - read = False + # metadata from file + meta = {} - data = [] - temp = [] + self.data.seek(0) + f = self.data.readlines() + count = 0 - for idx_row, row in enumerate(csv_reader): - if row == ['*****']: + for line in f: + count += 1 - if read == False: - read = True - else: - read = False - data.append(temp) + line = line.decode(encoding) - temp = [] + #remove whitespace + linesplit = line.strip() + linesplit = linesplit.split(splitsign) - continue + if len(linesplit) == 2: - if read: + key = linesplit[0].strip() + value = linesplit[1].split(';')[0].strip() - row = [r.replace(',', '.') for r in row] + meta[key] = value - temp.append(row) + if count >= skiprows: + break - #convert to pandas + # data + self.data.seek(0) + data = pd.read_csv(self.data, + encoding=encoding, + header=None, + skiprows=skiprows, + decimal=',', + thousands='.', + sep=';') - res = [] + data = data.iloc[2:] - freqs = [10.0, 5.0, 1.0, 0.1, 10.0] + ## add header to df + self.data.seek(0) + f = self.data.readlines() + count = 0 - for idx_data, d in enumerate(data): + for line in f: - t = pd.DataFrame(d[3:]) - t.columns = d[1] + line = line.decode(encoding) - freq = freqs[idx_data] - t['f'] = freq + count += 1 - for col in t.columns: - t[col] = pd.to_numeric(t[col]) + if count >= skiprows: + break - # add cycle number - dt = 1. / freq + #clean data + data = data.dropna(axis=1) - Nmax = int(np.ceil(t['ZEIT'].max() / dt)) + # add header from file + head = line.split(';') - N = np.zeros_like(t['ZEIT']) - - for i in range(Nmax): - if i == 0: - tmin = 0 - tmax = dt - else: - tmax = (i + 1) * dt - tmin = (i) * dt - - idx = t[(t['ZEIT'] >= tmin) & (t['ZEIT'] < tmax)].index - N[idx] = i - - t['N'] = N - - res.append(t) - - #remove second 10 Hz - res = pd.concat(res[:-1]) - - res['T'] = self.temperature - #res = res.sort_values(['f', 'ZEIT']) + data.columns = head + data.columns = [l.strip() for l in data.columns] #define in class - self.data = res.reset_index() + self.data = data + self.metadata.update(meta) + + # log infos + self._logger.info(self.metadata) + self._logger.info(self.data.head()) -class CITT_PTMDortmund(DataSineLoad): +class CITT_PTMDortmund(CITTBase): def _define_units(self): @@ -212,10 +485,7 @@ class CITT_PTMDortmund(DataSineLoad): time_idx = None assert time_idx is not None - temp['N'] = 0 - - #BUG: Ist in Messdatei falsch definiert und wird von PTM angepasst. ''' - #for cycle in range(Nfrom, Nto+1): + temp['N'] = 1 dt = 1.0 / frequency_test @@ -230,8 +500,8 @@ class CITT_PTMDortmund(DataSineLoad): #filter data idx = temp[(time_idx >= tmin) & (time_idx < tmax)].index - #set cycle number - temp.loc[idx, 'N'] = cycle + #set cycle number, cycle starts with 1 + temp.loc[idx, 'N'] = cycle + 1 cycle += 1 @@ -275,4 +545,202 @@ class CITT_PTMDortmund(DataSineLoad): # log infos self._logger.info(self.metadata) - self._logger.info(self.data.head()) \ No newline at end of file + self._logger.info(self.data.head()) + + +class CITT_UniSiegen(CITTBase): + + def _define_units(self): + + self.unit_s = 1 / 1000. #mm + self.unit_F = 1.0 #N + self.unit_t = 1. #s + + def update_parameter(self): + + self.meta_names_of_parameter = { + 'sigma': ['Oberspannung'], + 'f': ['Frequenz'], + 'T': ['Prüftemperatur'], + #'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'], + 'Nto': ['Maximale Zyklenanzahl'], + 't': ['Zeit'], + } #list of names + + self.data_column_names = { + 'time': ['Zeit'], + 'F': ['Kraft'], + 's_hor_1': ['Radialweg vorn'], + 's_hor_2': ['Radialweg hinten'], + 's_piston': ['Kolbenweg'], + 'N': ['Zyklenzähler'], + } + + def _process_data(self): + + meta, data = read_geosys(self.data, '015', metadata_ids=['003']) + + #define in class + self.data = data.reset_index() + self.metadata.update(meta) + + # log infos + self._logger.info(self.metadata) + self._logger.info(self.data.head()) + + +class CITT_BASt(CITTBase): + + def _define_units(self): + + self.unit_s = 1 / 1000. #mm + self.unit_F = 1.0 #N + self.unit_t = 1. #s + + def update_parameter(self): + + self.meta_names_of_parameter = { + 'sigma': ['Oberspannung'], + 'f': ['Versuchsart'], + 'T': ['Prüftemperatur\r\n'], + #'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'], + 'Nto': ['Maximale Zyklenanzahl'], + 't': ['Zeit'], + } #list of names + + self.data_column_names = { + 'time': ['Zeit'], + 'F': ['Vertikalkraft'], + 's_hor_1': ['Horizontalweg IWA vorn'], + 's_hor_2': ['Horizontalweg IWA hinten'], + 's_piston': ['Kolbenposition'], + 'N': ['Zyklenzähler'], + } + + def _process_data(self): + + meta, data = read_geosys(self.data, + '047', + metadata_ids=['003', '005', '015']) + + #define in class + self.data = data.reset_index() + self.metadata.update(meta) + + # log infos + self._logger.info(self.metadata) + self._logger.info(self.data.head()) + + def _modify_meta(self): + + s = self.metadata['f'] + s = s.split('Hz')[0].split('Steifigkeit')[-1].strip().replace(',', '.') + + self.metadata['f'] = float(s) + + s = self.metadata['T'] + s = s.split('°C')[0].strip().replace(',', '.') + + self.metadata['T'] = float(s) + + +class CITT_LaborHart(CITTBase): + + def _define_units(self): + + self.unit_s = 1.0 #mm + self.unit_F = 1.0 #N + self.unit_t = 1. / 1000.0 #s + + def update_parameter(self): + + self.meta_names_of_parameter = { + 'sigma': ['Oberspannung'], + 'T': ['Solltemperatur'], + 't': ['TIME'], + 'speciment_diameter': ['Probendurchmesser'], + 'speciment_height': ['Probenhöhe'], + } #list of names + + self.data_column_names = { + 'time': ['TIME'], + 'f': ['FREQUENZ'], + 'F': ['Load'], + 's_hor_1': ['SENSOR 4'], + 's_hor_2': ['SENSOR Extension'], + 's_piston': ['Position'], + 'N': ['Impulsnummer'], + } + + def _process_data(self): + + meta = {} + + splitsign = ':;' + encoding = 'latin-1' + skiprows = 14 + + self.data.seek(0) + f = self.data.readlines() + + count = 0 + + for line in f: + count += 1 + + #remove whitespace + line = line.decode(encoding) + linesplit = line.strip() + linesplit = linesplit.split(splitsign) + + if len(linesplit) == 2: + + meta[linesplit[0]] = linesplit[1] + + if count >= skiprows: + break + + # data + self.data.seek(0) + + data = pd.read_csv(self.data, + encoding=encoding, + skiprows=skiprows, + decimal=',', + sep=';') + + ## add header to df + self.data.seek(0) + f = self.data.readlines() + count = 0 + + for line in f: + count += 1 + + if count >= skiprows: + break + + line = line.decode(encoding) + head = line.split(';') + data.columns = head + + # FIX: Sigma nicht in Metadaten oder Messdaten enthalten + sigma = float( + os.path.split(self.filename)[-1].split('MPa')[0].strip().replace( + ',', '.')) + + meta['sigma'] = sigma + + #clean data + data = data.dropna(axis=1) + + #remove whitespace + data.columns = [c.strip() for c in data.columns] + + #define in class + self.data = data + self.metadata.update(meta) + + # log infos + self._logger.info(self.metadata) + self._logger.info(self.data.head())