markus/lib-paveit-demo
1
0
Fork 0
Code Pull Requests Activity

add more requests

Browse Source
This commit is contained in:
Markus Clauß
2023-06-02 08:53:44 +02:00
parent 4fe2c5f9fa
commit a27c94eb7c
10 changed files with 220 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/paveit/datamodels/citt.py
View File

@@ -34,7 +34,7 @@ class CyclicIndirectTensileTest(Document):
'index_background': True, 'index_background': True,
'index_cls': False, 'index_cls': False,
'auto_create_index': True, 'auto_create_index': True,
'collection': 'citt', 'collection': 'lab_citt',
"db_alias": 'dblabtests', "db_alias": 'dblabtests',
} }
@@ -152,25 +152,25 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
#optional parameter #optional parameter
## Piston ## Piston
s_hor_piston_amp = FloatField(required=False) s_piston_amp = FloatField(required=False)
s_hor_piston_freq = FloatField(required=False) s_piston_freq = FloatField(required=False)
s_hor_piston_phase = FloatField(required=False) s_piston_phase = FloatField(required=False)
s_hor_piston_offset = FloatField(required=False) s_piston_offset = FloatField(required=False)
s_hor_piston_slope = FloatField(required=False) s_piston_slope = FloatField(required=False)
s_hor_piston_r2 = FloatField(required=False) s_piston_r2 = FloatField(required=False)
s_hor_piston_cycle_min = ListField(FloatField(),required=False) s_piston_cycle_min = ListField(FloatField(),required=False)
s_hor_piston_min = FloatField(required=False) s_piston_min = FloatField(required=False)
s_hor_piston_min_std = FloatField(required=False) s_piston_min_std = FloatField(required=False)
s_hor_piston_min_diff_rel = FloatField(required=False) s_piston_min_diff_rel = FloatField(required=False)
s_hor_piston_cycle_max = ListField(FloatField(),required=False) s_piston_cycle_max = ListField(FloatField(),required=False)
s_hor_piston_max = FloatField(required=False) s_piston_max = FloatField(required=False)
s_hor_piston_max_std = FloatField(required=False) s_piston_max_std = FloatField(required=False)
s_hor_piston_max_dif_rel = FloatField(required=False) s_piston_max_diff_rel = FloatField(required=False)
s_hor_piston_cycle_mean = ListField(FloatField(),required=False) s_piston_cycle_mean = ListField(FloatField(),required=False)
s_hor_piston_mean = FloatField(required=False) s_piston_mean = FloatField(required=False)
s_hor_piston_mean_std = FloatField(required=False) s_piston_mean_std = FloatField(required=False)
s_hor_piston_mean_diff_rel = FloatField(required=False) s_piston_mean_diff_rel = FloatField(required=False)
s_hor_piston_cycle_diff = ListField(FloatField(),required=False) s_piston_cycle_diff = ListField(FloatField(),required=False)
s_hor_piston_diff = FloatField(required=False) s_piston_diff = FloatField(required=False)
s_hor_piston_diff_std = FloatField(required=False) s_piston_diff_std = FloatField(required=False)
s_hor_piston_diff_diff_rel = FloatField(required=False) s_piston_diff_diff_rel = FloatField(required=False)

4
src/paveit/datamodels/data.py
View File

@@ -23,7 +23,7 @@ class RawSinData(Document):
'index_background': True, 'index_background': True,
'index_cls': False, 'index_cls': False,
'auto_create_index': True, 'auto_create_index': True,
'collection': 'RawSinData', 'collection': 'data_rawsine',
"db_alias": 'dblabtests', "db_alias": 'dblabtests',
} }
@@ -40,7 +40,7 @@ class RawData(Document):
'index_background': True, 'index_background': True,
'index_cls': False, 'index_cls': False,
'auto_create_index': True, 'auto_create_index': True,
'collection': 'RawSinData', 'collection': 'data_raw',
"db_alias": 'dblabtests', "db_alias": 'dblabtests',
} }

7
src/paveit/datamodels/enumeration.py
View File

@@ -59,10 +59,13 @@ class Config(Document):
# convert data to dict # convert data to dict
data = fetch_recursive(self) data = fetch_recursive(self)
data = mongo_to_dict(data) data = mongo_to_dict(data)
print(data)
if enabled: if enabled:
data = list(filter(lambda item: item["enabled"], data)) if data['enabled']:
return data
else:
return []
return data return data
meta = { meta = {

46
src/paveit/datamodels/sheartest.py
View File

@@ -38,7 +38,7 @@ class DynamicShearTest(Document):
"db_alias": "db_alias":
'dblabtests', 'dblabtests',
'collection': 'collection':
'sheartest', 'lab_sheartest',
'indexes': [ 'indexes': [
[("lab", 1)], [("lab", 1)],
[("speciment_name", 1)], [("speciment_name", 1)],
@@ -213,25 +213,25 @@ class DynamicShearTestExtension(DynamicShearTest):
s_hor_2_diff_std = FloatField(required=False) s_hor_2_diff_std = FloatField(required=False)
s_hor_2_diff_diff_rel = FloatField(required=False) s_hor_2_diff_diff_rel = FloatField(required=False)
## Piston ## Piston
s_hor_piston_amp = FloatField(required=False) s_piston_amp = FloatField(required=False)
s_hor_piston_freq = FloatField(required=False) s_piston_freq = FloatField(required=False)
s_hor_piston_phase = FloatField(required=False) s_piston_phase = FloatField(required=False)
s_hor_piston_offset = FloatField(required=False) s_piston_offset = FloatField(required=False)
s_hor_piston_slope = FloatField(required=False) s_piston_slope = FloatField(required=False)
s_hor_piston_r2 = FloatField(required=False) s_piston_r2 = FloatField(required=False)
s_hor_piston_cycle_min = ListField(FloatField(),required=False) s_piston_cycle_min = ListField(FloatField(),required=False)
s_hor_piston_min = FloatField(required=False) s_piston_min = FloatField(required=False)
s_hor_piston_min_std = FloatField(required=False) s_piston_min_std = FloatField(required=False)
s_hor_piston_min_diff_rel = FloatField(required=False) s_piston_min_diff_rel = FloatField(required=False)
s_hor_piston_cycle_max = ListField(FloatField(),required=False) s_piston_cycle_max = ListField(FloatField(),required=False)
s_hor_piston_max = FloatField(required=False) s_piston_max = FloatField(required=False)
s_hor_piston_max_std = FloatField(required=False) s_piston_max_std = FloatField(required=False)
s_hor_piston_max_dif_rel = FloatField(required=False) s_piston_max_dif_rel = FloatField(required=False)
s_hor_piston_cycle_mean = ListField(FloatField(),required=False) s_piston_cycle_mean = ListField(FloatField(),required=False)
s_hor_piston_mean = FloatField(required=False) s_piston_mean = FloatField(required=False)
s_hor_piston_mean_std = FloatField(required=False) s_piston_mean_std = FloatField(required=False)
s_hor_piston_mean_diff_rel = FloatField(required=False) s_piston_mean_diff_rel = FloatField(required=False)
s_hor_piston_cycle_diff = ListField(FloatField(),required=False) s_piston_cycle_diff = ListField(FloatField(),required=False)
s_hor_piston_diff = FloatField(required=False) s_piston_diff = FloatField(required=False)
s_hor_piston_diff_std = FloatField(required=False) s_piston_diff_std = FloatField(required=False)
s_hor_piston_diff_diff_rel = FloatField(required=False) s_piston_diff_diff_rel = FloatField(required=False)

8
src/paveit/datamodels/taskmanager.py
View File

@@ -3,18 +3,20 @@ import datetime
from mongoengine import * from mongoengine import *
from .infrastructure import MachineBase from .machines import MachineBase
from .material import Material from .material import Material
from .project import Project from .project import Project
from .usermanagement import Organisation, User from .usermanagement import Organisation, User
from .workpackage import Workpackage from .workpackage import Workpackage
from .enumeration import ProcessStatusEnum from .enumeration import ProcessStatusEnum
from .client import Client
class TaskManagerBase(Document): class TaskManagerBase(Document):
org_id = LazyReferenceField(Organisation, required=True) org_id = LazyReferenceField(Organisation, required=True)
user_id = LazyReferenceField(User, required=True) user_id = LazyReferenceField(User, required=True)
client_id = LazyReferenceField(Client, required=True)
project_id = LazyReferenceField(Project, required=True) project_id = LazyReferenceField(Project, required=True)
wp_id = LazyReferenceField(Workpackage, required=False) wp_id = LazyReferenceField(Workpackage, required=False)
@@ -28,7 +30,7 @@ class TaskManagerBase(Document):
task_finished = DateTimeField(required=False) task_finished = DateTimeField(required=False)
assign_users = ListField(LazyReferenceField(User), required=False) assign_users = ListField(LazyReferenceField(User), required=False)
assign_machines = ListField(LazyReferenceField(MachineBase), required=False) assign_machine = LazyReferenceField(MachineBase, required=False)
series = StringField(default='Serie 01') series = StringField(default='Serie 01')
@@ -50,6 +52,6 @@ class TaskCITTStiffness(TaskManagerBase):
class TaskDynShearStiffness(TaskManagerBase): class TaskDynShearStiffness(TaskManagerBase):
material1 = LazyReferenceField(Material, required=True) material = LazyReferenceField(Material, required=True)
material2 = LazyReferenceField(Material, required=True) material2 = LazyReferenceField(Material, required=True)
bounding = LazyReferenceField(Material, required=True) bounding = LazyReferenceField(Material, required=True)

4
src/paveit/helper/minio.py
View File

@@ -9,7 +9,7 @@ def get_minio_client_processing(bucket_name = 'processing'):
os.environ["MINIO_URL"], os.environ["MINIO_URL"],
access_key=os.environ["MINIO_ACCESS_KEY"], access_key=os.environ["MINIO_ACCESS_KEY"],
secret_key=os.environ["MINIO_SECRET_KEY"], secret_key=os.environ["MINIO_SECRET_KEY"],
secure=False secure=True
) )
@@ -28,7 +28,7 @@ def get_minio_client_archive(bucket_name = 'archive'):
os.environ["MINIO_ARCHIVE_URL"], os.environ["MINIO_ARCHIVE_URL"],
access_key=os.environ["MINIO_ARCHIVE_ACCESS_KEY"], access_key=os.environ["MINIO_ARCHIVE_ACCESS_KEY"],
secret_key=os.environ["MINIO_ARCHIVE_SECRET_KEY"], secret_key=os.environ["MINIO_ARCHIVE_SECRET_KEY"],
secure=False secure=True
) )
found = client.bucket_exists(bucket_name) found = client.bucket_exists(bucket_name)

2
src/paveit/helper/mongo.py
View File

@@ -70,7 +70,7 @@ def mongo_get_results(resultsmodel, results: DataFrame, datamodel,
return True return True
def fetch_recursive(data, fetch_parameter=['norm', 'limits']): def fetch_recursive(data, fetch_parameter=['norm', 'limits', 'assign_machine']):
fields = data._fields fields = data._fields

51
src/paveit/labtest/base.py
View File

@@ -40,12 +40,6 @@ class DataSineLoad():
self._pre_run() self._pre_run()
def _which_machine(self):
"""
check the file and try to get the machine from the data
"""
pass
def _set_parameter(self): def _set_parameter(self):
self._logger.debug('run _set_parameter') self._logger.debug('run _set_parameter')
@@ -57,7 +51,7 @@ class DataSineLoad():
] ]
self.val_col_names = [ self.val_col_names = [
'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2' 'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2', 's_piston'
] ]
self.columns_analyse = [ self.columns_analyse = [
@@ -110,7 +104,7 @@ class DataSineLoad():
response.release_conn() response.release_conn()
self.data = io.BytesIO(self.data) self.data = io.BytesIO(self.data)
self._logger.debug('data read from s3') self._logger.debug('read data from s3')
def _calc_hash_of_bytesio(self): def _calc_hash_of_bytesio(self):
self._logger.debug('run _calc_hash_of_bytesio') self._logger.debug('run _calc_hash_of_bytesio')
@@ -145,6 +139,11 @@ class DataSineLoad():
for key, d in self.metadata.items(): for key, d in self.metadata.items():
try: try:
#remove units
for unit in ["°C", 'Hz']:
if unit in d:
d = d.split(unit)[0].strip()
f = float(d.replace(',', '.')) f = float(d.replace(',', '.'))
self.metadata[key] = f self.metadata[key] = f
except: except:
@@ -160,9 +159,27 @@ class DataSineLoad():
colnames = [sub.replace(name, par) for sub in colnames] colnames = [sub.replace(name, par) for sub in colnames]
self.data.columns = colnames self.data.columns = colnames
self._logger.debug(f'columns: {colnames}')
print(self.data.head())
print(self.data['s_piston'].head())
self._logger.debug(f'standardize_data: {self.data.columns}')
def _standardize_meta(self): def _standardize_meta(self):
self._logger.debug('run _standardize_meta') self._logger.debug('run _standardize_meta')
# remove "\r\n" ending from Windows
for col in list(self.metadata.keys()):
col_mod = col.replace('\r\n', '')
if col != col_mod:
self.metadata[col_mod] = self.metadata[col]
self.metadata.pop(col)
for par, names in self.meta_names_of_parameter.items(): for par, names in self.meta_names_of_parameter.items():
for name in names: for name in names:
@@ -172,6 +189,8 @@ class DataSineLoad():
self.metadata.pop(name) self.metadata.pop(name)
break break
self._logger.debug(f'meta (stand.): {self.metadata}')
def _modify_meta(self): def _modify_meta(self):
pass pass
@@ -184,13 +203,14 @@ class DataSineLoad():
# check if value in metadata: # check if value in metadata:
if name in self.metadata.keys(): if name in self.metadata.keys():
self._logger.error(f'add {name} from metadata to data')
self.data[name] = self.metadata[name] self.data[name] = self.metadata[name]
else: else:
self._logger.error(f'{name} not in data')
print(name)
raise raise
self._logger.debug(f'validate_data: {self.data.columns}')
def _validate_meta(self): def _validate_meta(self):
self._logger.debug('run _validate_meta') self._logger.debug('run _validate_meta')
@@ -295,7 +315,7 @@ class DataSineLoad():
gaps = idx_diff > (4 * dt_mean) gaps = idx_diff > (4 * dt_mean)
has_gaps = any(gaps) has_gaps = any(gaps)
if has_gaps == False: if has_gaps == False:
data_list.append(d) data_list.append(d)
@@ -359,8 +379,7 @@ class DataSineLoad():
self._calc_hash_of_bytesio() self._calc_hash_of_bytesio()
self._define_data_models() self._define_data_models()
self._data_in_db() #self._data_in_db()
self._which_machine()
self._set_parameter() self._set_parameter()
self.update_parameter() self.update_parameter()
self._define_units() self._define_units()
@@ -371,11 +390,11 @@ class DataSineLoad():
self._process_data() self._process_data()
self._meta_to_float() self._meta_to_float()
self._standardize_data()
self._standardize_meta() self._standardize_meta()
self._standardize_data()
self._modify_meta() self._modify_meta()
self._validate_data()
self._validate_meta() self._validate_meta()
self._validate_data()
self._post_string_to_float() self._post_string_to_float()
self._post_select_importent_columns() self._post_select_importent_columns()

149
src/paveit/labtest/citt.py
View File

@@ -13,6 +13,45 @@ from paveit.labtest import DataSineLoad
class CITTBase(DataSineLoad): class CITTBase(DataSineLoad):
def _set_parameter(self):
self._logger.debug('run _set_parameter')
self.split_data_based_on_parameter = ['T', 'sigma', 'f']
self.col_as_int = ['N']
self.col_as_float = [
'T', 'F', 's_piston', 's_hor_1', 'f', 's_hor_1', 's_hor_2'
]
self.val_col_names = [
'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2', 's_piston'
]
self.columns_analyse = [
'F', 's_hor_sum', 's_hor_1', 's_hor_2', 's_piston'
]
self.round_values = [('T', 3), ('sigma', 3)]
# Header names after standardization; check if exists
self.val_header_names = ['speciment_height', 'speciment_diameter']
self.number_of_load_cycles_for_analysis = 5
#Dummy Data, replace in Machine Config
self.meta_names_of_parameter = {
'sigma': ['Max. Spannung']
} #list of names
#Dummy Data, replace in Machine Config
self.data_column_names = {
'time': ['Time Series'],
'F': ['Load Series'],
's_hor_1': ['LVDT1 Series'],
's_hor_2': ['LVDT2 Series'],
}
def _sel_df(self, df, num=5, shift=-1): def _sel_df(self, df, num=5, shift=-1):
@@ -117,8 +156,10 @@ class CITTBase(DataSineLoad):
if data is None: continue if data is None: continue
if len(data) < 10: continue if len(data) < 10: continue
try: try:
self._logger.debug(f'run fit on subset {idx_data}')
data.index = data.index - data.index[0] data.index = data.index - data.index[0]
@@ -200,8 +241,11 @@ class CITTBase(DataSineLoad):
# TODO: Überarbeiten und erweitern (ISSUE #2) # TODO: Überarbeiten und erweitern (ISSUE #2)
res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[ res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[
'fit_s_hor_sum_phase'] 'fit_s_hor_sum_phase']
except: except Exception as e:
self._logger.exception(e)
res_temp = None res_temp = None
self._logger.debug(res_temp)
self.fit.append(res_temp) self.fit.append(res_temp)
@@ -266,54 +310,90 @@ class CITTBase(DataSineLoad):
s_hor_2=list(data['s_hor_2']), s_hor_2=list(data['s_hor_2']),
s_hor_sum=list(data['s_hor_sum']), s_hor_sum=list(data['s_hor_sum']),
) )
self._logger.debug(f'columns data, {data.columns}')
# add optional datas # add optional datas
for col in ['s_piston']: for col in ['s_piston']:
if col in data.columns: if col in data.columns:
self._logger.debug(f'add {col} to output data')
data_out[col] = list(data[col]) data_out[col] = list(data[col])
outkeys = list(data_out.keys())
self._logger.debug(f'write raw data to db, {outkeys}')
g = CITTSiffness(result=f.id, **data_out).save() g = CITTSiffness(result=f.id, **data_out).save()
class CITT_TUDresdenWille(CITTBase):
class CITT_TUDresden(CITTBase):
def _which_machine(self):
"""
check the file and try to get the machine from the data
"""
self._machine = 'TIRA'
def _define_units(self): def _define_units(self):
if self._machine == 'TIRA': self.unit_s = 1 / 1000. #mm
self.unit_s = 1 #mm self.unit_F = 1.0 #N
self.unit_F = 1. #N self.unit_t = 1. #s
self.unit_t = 1. #s
def update_parameter(self): def update_parameter(self):
if self._machine == 'TIRA': self.meta_names_of_parameter = {
'sigma': ['Oberspannung'],
'f': ['Prüffrequenz'],
'T': ['Prüftemperatur'],
't': ['Zeit'],
'speciment_diameter': ['Probekörberdurchmesser', 'Probekörberbreite'],
'speciment_height': ['Probekörperhöhe'],
} #list of names
self.meta_names_of_parameter = { self.data_column_names = {
'sigma': ['Oberspannung'], 'time': ['Zeit'],
'f': ['Sollfrequnz'], 'F': ['Kraft'],
'T': ['Solltemperatur'], 'N': ['Zyklenzähler'],
#'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'], 's_hor_1': ['Horizontalweg vorn', 'Weg Probe'],
#'Nto': ['Letzer Aufzeichnungslastwechsel', 'Last Cycle'], 's_hor_2': ['Horizontalweg hinten', 'Weg 2'],
't': ['Zeit'], 's_piston': ['Kolbenweg'],
'speciment_diameter': ['PK-Durchmesser'], }
'speciment_height': ['PK-Höhe'],
} #list of names
self.data_column_names = { def _process_data(self):
'time': ['Zeit'],
'F': ['Kraft'], meta, data = read_geosys(self.data, '039', metadata_ids=['001','003', '011', '019'])
'N': ['Lastwechsel'],
's_hor_1': ['IWA_1 2 mm'], #define in class
's_hor_2': ['IWA_2 2 mm'], self.data = data.reset_index()
's_piston': ['Kolbenweg'], self.metadata.update(meta)
}
# log infos
self._logger.debug(f'metadata: {self.metadata}')
self._logger.debug(f'data: {self.data.head()}')
print(data.head())
class CITT_TUDresdenTira(CITTBase):
def _define_units(self):
self.unit_s = 1 #mm
self.unit_F = 1. #N
self.unit_t = 1. #s
def update_parameter(self):
self.meta_names_of_parameter = {
'sigma': ['Oberspannung'],
'f': ['Sollfrequnz'],
'T': ['Solltemperatur'],
#'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'],
#'Nto': ['Letzer Aufzeichnungslastwechsel', 'Last Cycle'],
't': ['Zeit'],
'speciment_diameter': ['PK-Durchmesser', 'Probekörberbreite'],
'speciment_height': ['PK-Höhe', 'Probekörperhöhe'],
} #list of names
self.data_column_names = {
'time': ['Zeit'],
'F': ['Kraft'],
'N': ['Lastwechsel'],
's_hor_1': ['IWA_1 2 mm'],
's_hor_2': ['IWA_2 2 mm'],
's_piston': ['Kolbenweg'],
}
def _process_data(self): def _process_data(self):
@@ -396,6 +476,7 @@ class CITT_TUDresden(CITTBase):
self._logger.info(self.data.head()) self._logger.info(self.data.head())
class CITT_PTMDortmund(CITTBase): class CITT_PTMDortmund(CITTBase):
def _define_units(self): def _define_units(self):

12
src/paveit/labtest/sheartest.py
View File

@@ -233,6 +233,8 @@ class ShearTestExtension(ShearTest):
self._logger.info('run _calc') self._logger.info('run _calc')
self.fit = [] self.fit = []
errors = 0
for idx_data, data in enumerate(self.data): for idx_data, data in enumerate(self.data):
@@ -326,11 +328,13 @@ class ShearTestExtension(ShearTest):
self.fit.append(res_temp) self.fit.append(res_temp)
if (self.debug) & (len(self.fit) > 5): if (self.debug) & ( (len(self.fit) > 5) | (errors < 5) ):
break break
except Exception as e: except Exception as e:
self._logger.critical(e, exc_info=True) self._logger.critical(e, exc_info=True)
errors += 1
self.fit = pd.DataFrame.from_records(self.fit) self.fit = pd.DataFrame.from_records(self.fit)
@@ -445,13 +449,15 @@ class ShearTestExtensionTUDresden(ShearTestExtension):
self.unit_s = 1 / 1000. #mm self.unit_s = 1 / 1000. #mm
self.unit_F = 1.0 #N self.unit_F = 1.0 #N
self.unit_t = 1.0 #s self.unit_t = 1.0 #s
def update_parameter(self): def update_parameter(self):
self.meta_names_of_parameter = { self.meta_names_of_parameter = {
't': ['TIME'], 't': ['TIME'],
'speciment_diameter': ['Diameter of specimen\r\n'], 'speciment_diameter': ['Diameter of specimen'],
} #list of names } #list of names
self.data_column_names = { self.data_column_names = {