markus/lib-paveit-demo
1
0
Fork 0
Code Pull Requests Activity

Datenmodelle angepasst, einige Firmen in CITT übernommen

Browse Source
This commit is contained in:
Markus Clauß
2023-03-02 17:31:39 +01:00
parent e5c9f6904c
commit 1bbb560f31
14 changed files with 1421 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/paveit/datamodels/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
from .citt import *
from .data import *
from .material import *
from .project import *
from .sheartest import *
from .usermanagement import *
from .workpackage import *

102
src/paveit/datamodels/citt.py Normal file
View File

@@ -0,0 +1,102 @@
import datetime
from mongoengine import *
from .material import Material
from .project import Project
from .usermanagement import Organisation, User
from .workpackage import Workpackage
class CyclicIndirectTensileTest(Document):
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
standard = StringField(default='TP Asphalt Teil 24')
org_id = LazyReferenceField(Organisation, required=True)
project_id = LazyReferenceField(Project, required=True)
workpackage_id = LazyReferenceField(Workpackage, required=False)
user_id = LazyReferenceField(User,
required=True,
reverse_delete_rule=DO_NOTHING)
material = LazyReferenceField(Material, required=True)
tags = ListField(StringField())
machine = StringField(default=None)
filehash = StringField(required=True)
speciment_name = StringField()
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'citt',
"db_alias": 'dblabtests',
}
class CITTSiffnessResults(CyclicIndirectTensileTest):
#metadata
f_set = FloatField()
sigma_set = FloatField()
T_set = FloatField()
N_from = IntField()
N_to = IntField()
N_tot = IntField()
n_samples_per_cycle = IntField()
#results
stiffness = FloatField()
nu = FloatField()
phase = FloatField()
#required parameter
## F
F_amp = FloatField()
F_freq = FloatField()
F_phase = FloatField()
F_offset = FloatField()
F_slope = FloatField()
F_r2 = FloatField()
F_min = FloatField()
F_max = FloatField()
## S1
s_hor_1_amp = FloatField()
s_hor_1_freq = FloatField()
s_hor_1_phase = FloatField()
s_hor_1_offset = FloatField()
s_hor_1_slope = FloatField()
s_hor_1_r2 = FloatField()
s_hor_1_min = FloatField()
s_hor_1_max = FloatField()
## S2
s_hor_2_amp = FloatField()
s_hor_2_freq = FloatField()
s_hor_2_phase = FloatField()
s_hor_2_offset = FloatField()
s_hor_2_slope = FloatField()
s_hor_2_r2 = FloatField()
s_hor_2_min = FloatField()
s_hor_2_max = FloatField()
## S-Sum
s_hor_sum_amp = FloatField()
s_hor_sum_freq = FloatField()
s_hor_sum_phase = FloatField()
s_hor_sum_offset = FloatField()
s_hor_sum_slope = FloatField()
s_hor_sum_r2 = FloatField()
s_hor_sum_min = FloatField()
s_hor_sum_max = FloatField()
#optional parameter

57
src/paveit/datamodels/data.py Normal file
View File

@@ -0,0 +1,57 @@
import datetime
from mongoengine import *
from .citt import CyclicIndirectTensileTest
from .sheartest import DynamicShearTest
class RawData(Document):
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'rawdata',
"db_alias": 'dblabtests',
}
class DataSheartest(RawData):
#results
result_id = LazyReferenceField(DynamicShearTest,
required=True,
reverse_delete_rule=CASCADE)
# data
time = ListField(FloatField())
F = ListField(FloatField())
N = ListField(IntField())
s_vert_1 = ListField(FloatField())
s_vert_2 = ListField(FloatField())
s_vert_sum = ListField(FloatField(), required=False)
s_piston = ListField(FloatField(), required=False)
class CITTSiffness(RawData):
result = LazyReferenceField(CyclicIndirectTensileTest,
required=True,
reverse_delete_rule=CASCADE)
# data
time = ListField(FloatField())
F = ListField(FloatField())
N = ListField(IntField())
s_hor_1 = ListField(FloatField())
s_hor_2 = ListField(FloatField())
s_hor_sum = ListField(FloatField())
s_piston = ListField(FloatField(), required=False)

89
src/paveit/datamodels/material.py Normal file
View File

@@ -0,0 +1,89 @@
import datetime
from re import T
from mongoengine import *
from .project import Project
from .usermanagement import Organisation, User
class Material(Document):
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
org_id = LazyReferenceField(Organisation,
required=True,
reverse_delete_rule=CASCADE)
project_id = ListField(LazyReferenceField(Project,
required=False,
reverse_delete_rule=CASCADE),
required=True)
user_id = LazyReferenceField(User,
required=False,
reverse_delete_rule=DO_NOTHING)
tags = ListField(StringField())
norm = StringField(required=True, default='TP Asphalt Teil 24')
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'materials',
'indexes': [
[("material", 1)],
[("name", 1)],
]
}
class Asphalt(Material):
name = StringField()
material = StringField()
bitumen = StringField(required=False)
young_modulus = DictField()
fatigue = DictField()
class Bitumen(Material):
name = StringField()
material = StringField()
young_modulus = DictField()
class Bitumenemulsion(Material):
name = StringField()
material = StringField()
young_modulus = DictField()
class Epoxy(Material):
name = StringField()
material = StringField()
young_modulus = DictField()
class Kompaktasphalt(Material):
name = StringField()
class Dummy(Material):
name = StringField()
material = StringField()
young_modulus = DictField()

42
src/paveit/datamodels/project.py Normal file
View File

@@ -0,0 +1,42 @@
import datetime
from mongoengine import *
from .usermanagement import Organisation, User
class Project(Document):
name = StringField(required=True)
name_short = StringField(required=False)
project_id = StringField(required=True)
client = StringField(required=False)
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
org_id = LazyReferenceField(Organisation,
required=True,
reverse_delete_rule=CASCADE)
user_id = LazyReferenceField(User,
required=False,
reverse_delete_rule=DO_NOTHING)
tags = ListField(StringField())
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'projects',
'indexes': [
[("name_short", 1)],
[("name", 1)],
[("project_id", 1)],
]
}

107
src/paveit/datamodels/sheartest.py Normal file
View File

@@ -0,0 +1,107 @@
import datetime
from mongoengine import *
from .material import Material
from .project import Project
from .usermanagement import Organisation, User
from .workpackage import Workpackage
class DynamicShearTest(Document):
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
org_id = LazyReferenceField(Organisation, required=True)
project_id = LazyReferenceField(Project, required=True)
workpackage_id = LazyReferenceField(Workpackage, required=False)
user_id = LazyReferenceField(User,
required=True,
reverse_delete_rule=DO_NOTHING)
material1 = LazyReferenceField(Material, required=True)
material2 = LazyReferenceField(Material, required=True)
bounding = LazyReferenceField(Material, required=True)
gap_width = FloatField(default=1.0)
tags = ListField(StringField())
standard = StringField(default='TP Asphalt Teil 24')
machine = StringField(default=None)
filehash = StringField(required=True)
speciment_name = StringField()
meta = {
'allow_inheritance':
True,
'index_opts': {},
'index_background':
True,
'index_cls':
False,
'auto_create_index':
True,
'collection':
'sheartest',
'indexes': [
[("lab", 1)],
[("speciment_name", 1)],
[("project", 1)],
[("bruch", 1)],
[("lab", 1), ("project", 1)],
[("lab", 1), ("project", 1), ("workpackage", 1)],
[("lab", 1), ("project", 1), ("bounding", 1)],
]
}
class DynamicShearTestExtension(DynamicShearTest):
#metadata
f = FloatField(required=True)
sigma_normal = FloatField(required=True)
T = FloatField(required=True)
extension = FloatField(required=True)
stiffness = FloatField(required=True)
bruch = BooleanField(required=True)
#fit parameter
## F
fit_amp_F = FloatField(required=True)
fit_freq_F = FloatField(required=True)
fit_phase_F = FloatField(required=True)
fit_offset_F = FloatField(required=True)
fit_slope_F = FloatField(required=True)
## S1
fit_amp_s_vert_1 = FloatField(required=True)
fit_freq_s_vert_1 = FloatField(required=True)
fit_phase_s_vert_1 = FloatField(required=True)
fit_offset_s_vert_1 = FloatField(required=True)
fit_slope_s_vert_1 = FloatField(required=True)
r2_s_vert_1 = FloatField(required=True)
## S2
fit_amp_s_vert_2 = FloatField(required=True)
fit_freq_s_vert_2 = FloatField(required=True)
fit_phase_s_vert_2 = FloatField(required=True)
fit_offset_s_vert_2 = FloatField(required=True)
fit_slope_s_vert_2 = FloatField(required=True)
r2_s_vert_2 = FloatField(required=True)
## S-Sum
fit_amp_s_vert_sum = FloatField(required=True)
fit_freq_s_vert_sum = FloatField(required=True)
fit_phase_s_vert_sum = FloatField(required=True)
fit_offset_s_vert_sum = FloatField(required=True)
fit_slope_s_vert_sum = FloatField(required=True)
r2_s_vert_sum = FloatField(required=True)
## r2
r2_F = FloatField(required=True)
r2_s_vert_1 = FloatField(required=True)
r2_s_vert_2 = FloatField(required=True)
r2_s_vert_sum = FloatField(required=True)

51
src/paveit/datamodels/usermanagement.py Normal file
View File

@@ -0,0 +1,51 @@
import datetime
from mongoengine import *
class Organisation(Document):
name = StringField(required=True)
name_short = StringField(required=True)
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
labtest_citt = StringField(required=False)
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'organisation',
'db_alias': 'dbusers',
}
class User(Document):
active = BooleanField(required=True, default=True)
org_id = LazyReferenceField(Organisation,
required=True,
reverse_delete_rule=CASCADE)
date_added = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
name = StringField(required=True)
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'user'
}

33
src/paveit/datamodels/workpackage.py Normal file
View File

@@ -0,0 +1,33 @@
import datetime
from mongoengine import *
from .project import Project
from .usermanagement import User
class Workpackage(Document):
name = StringField(required=True)
name_short = StringField(required=False)
wp_id = StringField(required=True)
project_id = LazyReferenceField(Project, required=True)
user_id = LazyReferenceField(User,
required=False,
reverse_delete_rule=DO_NOTHING)
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'workpackages'
}

10
src/paveit/helper/__init__.py
View File

@@ -1,8 +1,10 @@
from .filehandling import read_file_to_bytesio
from .filehasher import calc_hash_of_bytes
from .minio import get_minio_client_archive, get_minio_client_processing
from .mongo import connect_mongo_dbs, mongo_get_results
__all__ = ['read_file_to_bytesio',
'get_minio_client_archive', 'get_minio_client_processing',
'calc_hash_of_bytes'
]
__all__ = [
'read_file_to_bytesio', 'connect_mongo_dbs', 'mongo_get_results',
'get_minio_client_processing', 'get_minio_client_archive',
'calc_hash_of_bytes'
]

97
src/paveit/helper/mongo.py Normal file
View File

@@ -0,0 +1,97 @@
from bson import ObjectId
from mongoengine import connect as mongo_connect
from pandas import DataFrame
def connect_mongo_db_labtests(username='admin',
password='admin',
host='mongo',
authentication_source='admin'):
mongo_connect('labtests',
username=username,
password=password,
host=host,
authentication_source=authentication_source,
alias='dblabtests')
def connect_mongo_db_usser(username='admin',
password='admin',
host='mongo',
authentication_source='admin'):
mongo_connect('users',
username=username,
password=password,
host=host,
authentication_source=authentication_source,
alias='dbusers')
def connect_mongo_dbs(username='admin',
password='admin',
host='mongo',
authentication_source='admin'):
connect_mongo_db_labtests(username=username,
password=password,
host=host,
authentication_source=authentication_source)
connect_mongo_db_usser(username=username,
password=password,
host=host,
authentication_source=authentication_source)
def mongo_upload_results(resultsmodel, results: DataFrame, datamodel,
data: DataFrame, filehash: str, org_id: ObjectId,
project_id: ObjectId, material_id: ObjectId,
user_id: ObjectId):
for idx, res in results.iterrows():
#upload results
meta['filehash'] = filehash
meta['org_id'] = org_id
meta['project_id'] = project_id
meta['material'] = material_id
meta['user_id'] = user_id
#check if result in db
#n = CITTSiffness.objects(**meta).count()
#print(n)
# write data
data_dict = res.to_dict()
data_dict.update(meta)
f = resultsmodel(**data_dict).save()
# upload data
data_sel = data[idx_fit]
# required data
data_out = dict(
time=data_sel.index,
F=list(data_sel['F']),
N=list(data_sel['N']),
s_hor_1=list(data_sel['s_hor_1']),
s_hor_2=list(data_sel['s_hor_2']),
s_hor_sum=list(data_sel['s_hor_sum']),
)
#optional data
for col in ['S_piston']:
if col in data_sel.columns:
data_out[col] = data_sel[col]
g = datamodel(result=f.id, **data_out).save()
def mongo_get_results(resultsmodel, results: DataFrame, datamodel,
data: DataFrame, filehash: str, org_id: ObjectId,
project_id: ObjectId, material_id: ObjectId,
user_id: ObjectId):
return True

3
src/paveit/io/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from .geosys import read_geosys
__all__ = ["read_geosys"]

227
src/paveit/io/geosys.py Normal file
View File

@@ -0,0 +1,227 @@
import csv
import os
from io import BytesIO
from sys import getsizeof
from numpy import array
from pandas import DataFrame
def detect_tabnum(filename, tabstr, encoding='utf-8'):
filename = os.path.normpath(filename)
tabstr = tabstr.lower()
#Einlesen
with open(filename, 'r', encoding=encoding) as inFile:
reader = csv.reader(inFile, delimiter='\t')
counter = 0
for row in reader:
row = [r.lower() for r in row]
if any(tabstr in mystring for mystring in row):
if 'plain' in row:
return row[1]
counter += 1
if counter > 100:
return False
def str2float(str):
try:
str = str.replace(',', '.')
return float(str)
except:
return None
def read_geosys(buffer: BytesIO,
table,
pkdata='001',
metadata_ids=['003', '015'],
encoding='utf-8',
to_si=False,
debug=False):
'''
:param buffer: Bytes IO Object
:param table: Table-Number
:param pkdata: Table-Number of speciment definitions, default: 1
:param encoding: Encoding, default: utf-8
:param debug: debug-mode
:return:
'''
try:
dictOut = {}
dictOut['durch'] = 0
dictOut['hoehe'] = 0
#---------------------------------------------------------------------
#Daten einlesen und umwandeln
#---------------------------------------------------------------------
data = []
#Einlesen
buffer.seek(0)
lines_file = buffer.readlines()
for line in lines_file:
try:
line = line.decode(encoding)
line = line.split('\t')
if len(line) > 2:
data.append(line)
except:
pass
if debug:
print('Anz. Datensätze: ', str(len(data)), getsizeof(data))
#aufräumen
##Datenstruktur anlegen
data_processed = {}
data_processed['head'] = []
data_processed['metadata'] = {}
data_processed['data'] = []
for i in metadata_ids:
data_processed['metadata'][i] = []
for idx, d in enumerate(data):
try:
v = d[0][0:3]
if v in pkdata: data_processed['head'].append(d)
if v in metadata_ids: data_processed['metadata'][v].append(d)
if v in table: data_processed['data'].append(d)
except:
pass
# replace object
data = data_processed
if debug:
print('data_clean fin')
## Header aufbereiten
for idx, row in enumerate(data['head']):
if idx == 0:
id_durchmesser = None
id_hoehe = None
id_name = None
for idx_name, name in enumerate(row):
name_lower = name.lower()
if any(map(name_lower.__contains__, ['durchmesser'])):
id_durchmesser = idx_name
elif any(map(name_lower.__contains__, ['bezeichnung'])):
id_name = idx_name
elif any(map(name_lower.__contains__, ['höhe'])):
id_hoehe = idx_name
if debug:
print(id_durchmesser, id_hoehe, id_name)
elif idx == 1:
unit_durch = None
unit_hoehe = None
try:
unit_durch = row[id_durchmesser]
unit_hoehe = row[id_hoehe]
except:
pass
elif idx == 2:
durchmesser = None
hoehe = None
name = None
try:
durchmesser = str2float(row[id_durchmesser])
hoehe = str2float(row[id_hoehe])
name = row[id_name]
except:
pass
header = {
'speciment_diameter': durchmesser,
'speciment_height': hoehe,
'name': name,
'unit_h': unit_hoehe,
'unit_d': unit_durch
}
meta = data['metadata']
for key in meta.keys():
sel = meta[key]
assert len(sel[0]) == len(sel[2])
header_append = {
sel[0][i]: sel[2][i].strip()
for i in range(len(sel[0]))
}
header.update(header_append)
if debug:
print('header\n', header)
# add metadata to header
## Daten in Pandas DataFrame umwandeln
if debug:
print('daten umwandel')
temp = []
for idx, row in enumerate(data['data']):
if idx == 0:
if debug:
print('head')
data_head = []
for idx_name, name in enumerate(row):
if idx_name <= 1: continue
data_head.append(name)
elif idx == 1:
data_units = []
for idx_name, name in enumerate(row):
if idx_name <= 1: continue
data_units.append(name)
else:
t = []
for idx_col, value in enumerate(row):
if idx_col <= 1:
continue
else:
t.append(str2float(value))
temp.append(t)
data = array(temp)
if debug:
print(data_head, data_units)
## Bezeichnungen der Daten normalisieren
# Pandas DataFrame erstellen
data = DataFrame(data=data, columns=data_head)
if debug:
print(data.head())
return header, data
except:
print('Fehler beim lesen')
raise

183
src/paveit/labtest/base.py
View File

@@ -40,6 +40,12 @@ class DataSineLoad():
self._pre_run()
def _which_machine(self):
"""
check the file and try to get the machine from the data
"""
pass
def _set_parameter(self):
self._logger.debug('run _set_parameter')
@@ -120,6 +126,15 @@ class DataSineLoad():
encoding = 'utf-8'
self.data = pd.read_csv(self.data, encoding=encoding)
def _meta_to_float(self):
for key, d in self.metadata.items():
try:
f = float(d.replace(',', '.'))
self.metadata[key] = f
except:
pass
def _standardize_data(self):
self._logger.debug('run _standardize_data')
@@ -143,12 +158,24 @@ class DataSineLoad():
break
def _modify_meta(self):
pass
def _validate_data(self):
self._logger.debug('run _validate_data')
for name in self.val_col_names:
if not name in self.data.columns:
raise
# check if value in metadata:
if name in self.metadata.keys():
self.data[name] = self.metadata[name]
else:
print(name)
raise
def _validate_meta(self):
self._logger.debug('run _validate_meta')
@@ -157,6 +184,17 @@ class DataSineLoad():
if not name in self.metadata:
raise
def _post_string_to_float(self):
sel = self.data.select_dtypes(include=['object'])
if sel.empty:
return
for col in sel.columns:
self.data[col] = pd.to_numeric(self.data[col].str.replace(
',', '.'))
def _post_apply_units(self):
for col in ['s_hor_sum', 's_hor_1', 's_hor_2']:
@@ -209,6 +247,13 @@ class DataSineLoad():
for idx, d in data_gp:
if d.empty: continue
if any(d['f'] <= 0.0): continue
#reset N
d['N'] = d['N'] - d['N'].iloc[0] + 1
idx_diff = np.diff(d.index)
dt_mean = idx_diff.mean()
@@ -249,129 +294,27 @@ class DataSineLoad():
(a): Based on window of TP-Asphalt
(b) last N cycles
DUMMY FUNCTION
"""
self._logger.debug('run _fit_select_data')
def sel_df(df, num=5):
N = df['N'].unique()
freq = float(df['f'].unique()[0])
# define cycles to select
if freq == 10.0:
Nfrom = 98
Nto = 103
elif freq == 5.0:
Nfrom = 93
Nto = 97
elif freq == 3.0:
Nfrom = 43
Nto = 47
elif freq == 1.0:
Nfrom = 13
Nto = 17
elif freq == 0.3:
Nfrom = 8
Nto = 12
elif freq == 0.1:
Nfrom = 3
Nto = 7
else:
Nfrom = None
Nto = None
# Fall 1: nicht alle LW in Datei
if (max(N) < Nto) & (len(N) >= num):
df_sel = df[(df['N'] >= N[-num]) & (df['N'] <= N[-1])]
# Fall 2:
else:
if Nfrom != None:
if len(N) > Nto - Nfrom:
df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)]
return df_sel
if not isinstance(self.data, list):
if self.number_of_load_cycles_for_analysis > 1:
df_sel = [
sel_df(self.data,
num=self.number_of_load_cycles_for_analysis)
]
else:
df_sel = [self.data]
else:
df_sel = []
for d in self.data:
if self.number_of_load_cycles_for_analysis > 1:
d_sel = sel_df(d,
num=self.number_of_load_cycles_for_analysis)
else:
d_sel = d
df_sel.append(d_sel)
# replace data
self.data = df_sel
pass
def _calc(self):
"""
Calculate Results
DUMMY FUNCTION
"""
self.fit = []
for idx_data, data in enumerate(self.data):
self._logger.info('run _calc base')
print('run BASE')
if data is None: continue
if len(data) < 10: continue
def save(self):
'''
save results to database
DUMMY FUNCTION
'''
data.index = data.index - data.index[0]
res_temp = {}
x = data.index.values
freq = np.round(float(data['f'].unique()), 2)
sigma = float(data['sigma'].unique())
temperature = float(data['T'].unique())
for idxcol, col in enumerate(self.columns_analyse):
if not col in data.columns: continue
y = data[col].values
res = fit_cos(x, y, freq=freq)
for key, value in res.items():
res_temp[f'fit_{col}_{key}'] = value
res_temp[f'fit_{col}_max'] = max(y)
res_temp[f'fit_{col}_min'] = min(y)
res_temp['f'] = freq
res_temp['sigma'] = sigma
res_temp['T'] = temperature
## Stiffness
deltaF = res_temp['fit_F_amp']
nu = calc_nu(temperature)
res_temp['nu'] = nu
h = float(self.metadata['speciment_height'])
deltaU = res_temp['fit_s_hor_sum_amp']
res_temp['E'] = (deltaF * (0.274 + nu)) / (h * deltaU)
self.fit.append(res_temp)
self.fit = pd.DataFrame.from_records(self.fit)
self.fit = self.fit.set_index(['T', 'f', 'sigma'])
nsamples = len(self.fit)
self._logger.info(f'fitting finished, add {nsamples} samples')
self._logger.debug(self.fit['E'])
pass
def _pre_run(self):
@@ -379,6 +322,7 @@ class DataSineLoad():
self._read_from_s3_to_bytesio()
self._calc_hash_of_bytesio()
self._which_machine()
self._set_parameter()
self.update_parameter()
self._define_units()
@@ -387,12 +331,15 @@ class DataSineLoad():
self._logger.info('run task')
self._process_data()
self._meta_to_float()
self._standardize_data()
self._standardize_meta()
self._modify_meta()
self._validate_data()
self._validate_meta()
self._post_string_to_float()
self._post_select_importent_columns()
self._post_apply_units()
self._post_calc_missiong_values()

602
src/paveit/labtest/citt.py
View File

@@ -4,101 +4,374 @@ from csv import reader
import numpy as np
import pandas as pd
from bson import ObjectId
from paveit import calc_nu, fit_cos
from paveit.datamodels import CITTSiffness, CITTSiffnessResults
from paveit.io import read_geosys
from paveit.labtest import DataSineLoad
class CITTBase(DataSineLoad):
def _calc(self):
return (self.df.mean().mean(), self.df.max().max())
def _sel_df(self, df, num=5, shift=-1):
N = df['N'].unique()
n_N = len(N)
max_N = max(N)
min_N = min(N)
class CITT_KIT(DataSineLoad):
freq = float(df['f'].unique()[0])
# define cycles to select
if freq == 10.0:
Nfrom = 98
Nto = 103
elif freq == 5.0:
Nfrom = 93
Nto = 97
elif freq == 3.0:
Nfrom = 43
Nto = 47
elif freq == 1.0:
Nfrom = 13
Nto = 17
elif freq == 0.3:
Nfrom = 8
Nto = 12
elif freq == 0.1:
Nfrom = 3
Nto = 7
else:
Nfrom = None
Nto = None
# Fall 1: nur num Lastwechsel
if n_N < num:
df_sel = None
elif n_N == num:
df_sel = df
# Fall 2: nicht alle LW in Datei
elif (max_N < Nto) & (n_N > num):
df_sel = df[(df['N'] >= N[-num + shift])
& (df['N'] <= N[-1 + shift])]
# Fall 3: Auswahl wie oben definiert
elif (Nfrom >= min_N) & (Nto < max_N):
df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)]
# Fall 4: Auswahl unbekannt
else:
df_sel = None
return df_sel
def _fit_select_data(self):
"""
select N load cycles from original data
(a): Based on window of TP-Asphalt
(b) last N cycles
"""
self._logger.debug('run _fit_select_data')
self.max_N_in_data = []
if not isinstance(self.data, list):
if self.number_of_load_cycles_for_analysis > 1:
self.max_N_in_data.append(self.data['N'].max())
df_sel = [
self._sel_df(self.data,
num=self.number_of_load_cycles_for_analysis)
]
else:
df_sel = [self.data]
else:
df_sel = []
for d in self.data:
self.max_N_in_data.append(d['N'].max())
if self.number_of_load_cycles_for_analysis > 1:
d_sel = self._sel_df(
d, num=self.number_of_load_cycles_for_analysis)
else:
d_sel = d
df_sel.append(d_sel)
# replace data
self.data = df_sel
def _calc(self):
return (self.df.mean().mean(), self.df.max().max())
self._logger.info('run _calc CITT')
print('run CITT')
self.fit = []
for idx_data, data in enumerate(self.data):
if data is None: continue
if len(data) < 10: continue
try:
data.index = data.index - data.index[0]
res_temp = {}
res_temp['idx'] = idx_data
x = data.index.values
freq = np.round(float(data['f'].unique()), 2)
sigma = float(data['sigma'].unique())
temperature = float(data['T'].unique())
for idxcol, col in enumerate(self.columns_analyse):
if not col in data.columns: continue
y = data[col].values
res = fit_cos(x, y, freq=freq)
for key, value in res.items():
res_temp[f'fit_{col}_{key}'] = value
res_temp[f'fit_{col}_max'] = max(y)
res_temp[f'fit_{col}_min'] = min(y)
# add more metadata
res_temp['f_set'] = freq
res_temp['sigma_set'] = sigma
res_temp['T_set'] = temperature
res_temp['N_from'] = data['N'].min()
res_temp['N_to'] = data['N'].max()
res_temp['N_tot'] = self.max_N_in_data[idx_data]
res_temp['n_samples_per_cycle'] = int(
len(data) / (res_temp['N_to'] - res_temp['N_from'] + 1))
## Stiffness
deltaF = res_temp['fit_F_amp']
nu = calc_nu(temperature)
res_temp['nu'] = nu
h = float(self.metadata['speciment_height'])
deltaU = res_temp['fit_s_hor_sum_amp']
res_temp['stiffness'] = (deltaF * (0.274 + nu)) / (h * deltaU)
# TODO: Überarbeiten und erweitern (ISSUE #2)
res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[
'fit_s_hor_sum_phase']
except:
res_temp = None
self.fit.append(res_temp)
self.fit = pd.DataFrame.from_records(self.fit)
self.fit = self.fit.reset_index(drop=True).set_index('idx')
#self.fit = self.fit.set_index(['T', 'f', 'sigma'])
nsamples = len(self.fit)
self._logger.info(f'fitting finished, add {nsamples} samples')
self._logger.debug(self.fit['stiffness'])
def save(self,
org_id: ObjectId,
project_id: ObjectId,
material_id: ObjectId,
user_id: ObjectId,
meta: dict = {},
wp_id: ObjectId | None = None):
"""
save results to mongodb
"""
if not hasattr(self, 'fit'):
raise
# precheck data and results
assert len(self.data) == len(self.fit)
for idx_fit, fit in self.fit.iterrows():
data = self.data[idx_fit]
meta['filehash'] = self.filehash
meta['org_id'] = org_id
meta['project_id'] = project_id
meta['material'] = material_id
meta['user_id'] = user_id
#check if result in db
#n = CITTSiffness.objects(**meta).count()
#print(n)
# write data
data_dict = fit.to_dict()
data_dict.update(meta)
# remove 'fit_' from keys:
for key in list(data_dict.keys()):
if key.startswith('fit_'):
data_dict[key[4:]] = data_dict[key]
data_dict.pop(key)
# rename fields
def rename_field(d, old, new):
d[new] = d[old]
d.pop(old)
f = CITTSiffnessResults(**data_dict).save()
# required data
data_out = dict(
time=data.index,
F=list(data['F']),
N=list(data['N']),
s_hor_1=list(data['s_hor_1']),
s_hor_2=list(data['s_hor_2']),
s_hor_sum=list(data['s_hor_sum']),
)
# add optional datas
for col in ['s_piston']:
if col in data.columns:
data_out[col] = list(data[col])
g = CITTSiffness(result=f.id, **data_out).save()
class CITT_TUDresden(CITTBase):
def _which_machine(self):
"""
check the file and try to get the machine from the data
"""
self._machine = 'TIRA'
def _define_units(self):
if self._machine == 'TIRA':
self.unit_s = 1 #mm
self.unit_F = 1. #N
self.unit_t = 1. #s
def update_parameter(self):
if self._machine == 'TIRA':
self.meta_names_of_parameter = {
'sigma': ['Oberspannung'],
'f': ['Sollfrequnz'],
'T': ['Solltemperatur'],
#'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'],
#'Nto': ['Letzer Aufzeichnungslastwechsel', 'Last Cycle'],
't': ['Zeit'],
'speciment_diameter': ['PK-Durchmesser'],
'speciment_height': ['PK-Höhe'],
} #list of names
self.data_column_names = {
'time': ['Zeit'],
'F': ['Kraft'],
'N': ['Lastwechsel'],
's_hor_1': ['IWA_1 2 mm'],
's_hor_2': ['IWA_2 2 mm'],
's_piston': ['Kolbenweg'],
}
def _process_data(self):
self._logger.debug('convert bytes to pandas.DataFrame')
self.data.seek(0)
with io.TextIOWrapper(self.data, encoding='latin-1') as read_obj:
csv_reader = reader(read_obj, delimiter=';')
# -----------------------------------------------------------------------------------------------
# TIRA
# -----------------------------------------------------------------------------------------------
if self._machine == 'TIRA':
encoding = 'latin-1'
skiprows = 29
hasunits = True
splitsign = ':;'
read = False
# metadata from file
meta = {}
data = []
temp = []
self.data.seek(0)
f = self.data.readlines()
count = 0
for idx_row, row in enumerate(csv_reader):
if row == ['*****']:
for line in f:
count += 1
if read == False:
read = True
else:
read = False
data.append(temp)
line = line.decode(encoding)
temp = []
#remove whitespace
linesplit = line.strip()
linesplit = linesplit.split(splitsign)
continue
if len(linesplit) == 2:
if read:
key = linesplit[0].strip()
value = linesplit[1].split(';')[0].strip()
row = [r.replace(',', '.') for r in row]
meta[key] = value
temp.append(row)
if count >= skiprows:
break
#convert to pandas
# data
self.data.seek(0)
data = pd.read_csv(self.data,
encoding=encoding,
header=None,
skiprows=skiprows,
decimal=',',
thousands='.',
sep=';')
res = []
data = data.iloc[2:]
freqs = [10.0, 5.0, 1.0, 0.1, 10.0]
## add header to df
self.data.seek(0)
f = self.data.readlines()
count = 0
for idx_data, d in enumerate(data):
for line in f:
t = pd.DataFrame(d[3:])
t.columns = d[1]
line = line.decode(encoding)
freq = freqs[idx_data]
t['f'] = freq
count += 1
for col in t.columns:
t[col] = pd.to_numeric(t[col])
if count >= skiprows:
break
# add cycle number
dt = 1. / freq
#clean data
data = data.dropna(axis=1)
Nmax = int(np.ceil(t['ZEIT'].max() / dt))
# add header from file
head = line.split(';')
N = np.zeros_like(t['ZEIT'])
for i in range(Nmax):
if i == 0:
tmin = 0
tmax = dt
else:
tmax = (i + 1) * dt
tmin = (i) * dt
idx = t[(t['ZEIT'] >= tmin) & (t['ZEIT'] < tmax)].index
N[idx] = i
t['N'] = N
res.append(t)
#remove second 10 Hz
res = pd.concat(res[:-1])
res['T'] = self.temperature
#res = res.sort_values(['f', 'ZEIT'])
data.columns = head
data.columns = [l.strip() for l in data.columns]
#define in class
self.data = res.reset_index()
self.data = data
self.metadata.update(meta)
# log infos
self._logger.info(self.metadata)
self._logger.info(self.data.head())
class CITT_PTMDortmund(DataSineLoad):
class CITT_PTMDortmund(CITTBase):
def _define_units(self):
@@ -212,10 +485,7 @@ class CITT_PTMDortmund(DataSineLoad):
time_idx = None
assert time_idx is not None
temp['N'] = 0
#BUG: Ist in Messdatei falsch definiert und wird von PTM angepasst. '''
#for cycle in range(Nfrom, Nto+1):
temp['N'] = 1
dt = 1.0 / frequency_test
@@ -230,8 +500,8 @@ class CITT_PTMDortmund(DataSineLoad):
#filter data
idx = temp[(time_idx >= tmin) & (time_idx < tmax)].index
#set cycle number
temp.loc[idx, 'N'] = cycle
#set cycle number, cycle starts with 1
temp.loc[idx, 'N'] = cycle + 1
cycle += 1
@@ -275,4 +545,202 @@ class CITT_PTMDortmund(DataSineLoad):
# log infos
self._logger.info(self.metadata)
self._logger.info(self.data.head())
self._logger.info(self.data.head())
class CITT_UniSiegen(CITTBase):
def _define_units(self):
self.unit_s = 1 / 1000. #mm
self.unit_F = 1.0 #N
self.unit_t = 1. #s
def update_parameter(self):
self.meta_names_of_parameter = {
'sigma': ['Oberspannung'],
'f': ['Frequenz'],
'T': ['Prüftemperatur'],
#'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'],
'Nto': ['Maximale Zyklenanzahl'],
't': ['Zeit'],
} #list of names
self.data_column_names = {
'time': ['Zeit'],
'F': ['Kraft'],
's_hor_1': ['Radialweg vorn'],
's_hor_2': ['Radialweg hinten'],
's_piston': ['Kolbenweg'],
'N': ['Zyklenzähler'],
}
def _process_data(self):
meta, data = read_geosys(self.data, '015', metadata_ids=['003'])
#define in class
self.data = data.reset_index()
self.metadata.update(meta)
# log infos
self._logger.info(self.metadata)
self._logger.info(self.data.head())
class CITT_BASt(CITTBase):
def _define_units(self):
self.unit_s = 1 / 1000. #mm
self.unit_F = 1.0 #N
self.unit_t = 1. #s
def update_parameter(self):
self.meta_names_of_parameter = {
'sigma': ['Oberspannung'],
'f': ['Versuchsart'],
'T': ['Prüftemperatur\r\n'],
#'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'],
'Nto': ['Maximale Zyklenanzahl'],
't': ['Zeit'],
} #list of names
self.data_column_names = {
'time': ['Zeit'],
'F': ['Vertikalkraft'],
's_hor_1': ['Horizontalweg IWA vorn'],
's_hor_2': ['Horizontalweg IWA hinten'],
's_piston': ['Kolbenposition'],
'N': ['Zyklenzähler'],
}
def _process_data(self):
meta, data = read_geosys(self.data,
'047',
metadata_ids=['003', '005', '015'])
#define in class
self.data = data.reset_index()
self.metadata.update(meta)
# log infos
self._logger.info(self.metadata)
self._logger.info(self.data.head())
def _modify_meta(self):
s = self.metadata['f']
s = s.split('Hz')[0].split('Steifigkeit')[-1].strip().replace(',', '.')
self.metadata['f'] = float(s)
s = self.metadata['T']
s = s.split('°C')[0].strip().replace(',', '.')
self.metadata['T'] = float(s)
class CITT_LaborHart(CITTBase):
def _define_units(self):
self.unit_s = 1.0 #mm
self.unit_F = 1.0 #N
self.unit_t = 1. / 1000.0 #s
def update_parameter(self):
self.meta_names_of_parameter = {
'sigma': ['Oberspannung'],
'T': ['Solltemperatur'],
't': ['TIME'],
'speciment_diameter': ['Probendurchmesser'],
'speciment_height': ['Probenhöhe'],
} #list of names
self.data_column_names = {
'time': ['TIME'],
'f': ['FREQUENZ'],
'F': ['Load'],
's_hor_1': ['SENSOR 4'],
's_hor_2': ['SENSOR Extension'],
's_piston': ['Position'],
'N': ['Impulsnummer'],
}
def _process_data(self):
meta = {}
splitsign = ':;'
encoding = 'latin-1'
skiprows = 14
self.data.seek(0)
f = self.data.readlines()
count = 0
for line in f:
count += 1
#remove whitespace
line = line.decode(encoding)
linesplit = line.strip()
linesplit = linesplit.split(splitsign)
if len(linesplit) == 2:
meta[linesplit[0]] = linesplit[1]
if count >= skiprows:
break
# data
self.data.seek(0)
data = pd.read_csv(self.data,
encoding=encoding,
skiprows=skiprows,
decimal=',',
sep=';')
## add header to df
self.data.seek(0)
f = self.data.readlines()
count = 0
for line in f:
count += 1
if count >= skiprows:
break
line = line.decode(encoding)
head = line.split(';')
data.columns = head
# FIX: Sigma nicht in Metadaten oder Messdaten enthalten
sigma = float(
os.path.split(self.filename)[-1].split('MPa')[0].strip().replace(
',', '.'))
meta['sigma'] = sigma
#clean data
data = data.dropna(axis=1)
#remove whitespace
data.columns = [c.strip() for c in data.columns]
#define in class
self.data = data
self.metadata.update(meta)
# log infos
self._logger.info(self.metadata)
self._logger.info(self.data.head())