markus/lib-paveit-demo
1
0
Fork 0
Code Pull Requests Activity

add more requests

Browse Source
This commit is contained in:
Markus Clauß
2023-06-02 08:53:44 +02:00
parent 4fe2c5f9fa
commit a27c94eb7c
10 changed files with 220 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/paveit/datamodels/citt.py
View File

@@ -34,7 +34,7 @@ class CyclicIndirectTensileTest(Document):
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'citt',
'collection': 'lab_citt',
"db_alias": 'dblabtests',
}
@@ -152,25 +152,25 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
#optional parameter
## Piston
s_hor_piston_amp = FloatField(required=False)
s_hor_piston_freq = FloatField(required=False)
s_hor_piston_phase = FloatField(required=False)
s_hor_piston_offset = FloatField(required=False)
s_hor_piston_slope = FloatField(required=False)
s_hor_piston_r2 = FloatField(required=False)
s_hor_piston_cycle_min = ListField(FloatField(),required=False)
s_hor_piston_min = FloatField(required=False)
s_hor_piston_min_std = FloatField(required=False)
s_hor_piston_min_diff_rel = FloatField(required=False)
s_hor_piston_cycle_max = ListField(FloatField(),required=False)
s_hor_piston_max = FloatField(required=False)
s_hor_piston_max_std = FloatField(required=False)
s_hor_piston_max_dif_rel = FloatField(required=False)
s_hor_piston_cycle_mean = ListField(FloatField(),required=False)
s_hor_piston_mean = FloatField(required=False)
s_hor_piston_mean_std = FloatField(required=False)
s_hor_piston_mean_diff_rel = FloatField(required=False)
s_hor_piston_cycle_diff = ListField(FloatField(),required=False)
s_hor_piston_diff = FloatField(required=False)
s_hor_piston_diff_std = FloatField(required=False)
s_hor_piston_diff_diff_rel = FloatField(required=False)
s_piston_amp = FloatField(required=False)
s_piston_freq = FloatField(required=False)
s_piston_phase = FloatField(required=False)
s_piston_offset = FloatField(required=False)
s_piston_slope = FloatField(required=False)
s_piston_r2 = FloatField(required=False)
s_piston_cycle_min = ListField(FloatField(),required=False)
s_piston_min = FloatField(required=False)
s_piston_min_std = FloatField(required=False)
s_piston_min_diff_rel = FloatField(required=False)
s_piston_cycle_max = ListField(FloatField(),required=False)
s_piston_max = FloatField(required=False)
s_piston_max_std = FloatField(required=False)
s_piston_max_diff_rel = FloatField(required=False)
s_piston_cycle_mean = ListField(FloatField(),required=False)
s_piston_mean = FloatField(required=False)
s_piston_mean_std = FloatField(required=False)
s_piston_mean_diff_rel = FloatField(required=False)
s_piston_cycle_diff = ListField(FloatField(),required=False)
s_piston_diff = FloatField(required=False)
s_piston_diff_std = FloatField(required=False)
s_piston_diff_diff_rel = FloatField(required=False)

4
src/paveit/datamodels/data.py
View File

@@ -23,7 +23,7 @@ class RawSinData(Document):
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'RawSinData',
'collection': 'data_rawsine',
"db_alias": 'dblabtests',
}
@@ -40,7 +40,7 @@ class RawData(Document):
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'RawSinData',
'collection': 'data_raw',
"db_alias": 'dblabtests',
}

7
src/paveit/datamodels/enumeration.py
View File

@@ -59,10 +59,13 @@ class Config(Document):
# convert data to dict
data = fetch_recursive(self)
data = mongo_to_dict(data)
print(data)
if enabled:
data = list(filter(lambda item: item["enabled"], data))
if data['enabled']:
return data
else:
return []
return data
meta = {

46
src/paveit/datamodels/sheartest.py
View File

@@ -38,7 +38,7 @@ class DynamicShearTest(Document):
"db_alias":
'dblabtests',
'collection':
'sheartest',
'lab_sheartest',
'indexes': [
[("lab", 1)],
[("speciment_name", 1)],
@@ -213,25 +213,25 @@ class DynamicShearTestExtension(DynamicShearTest):
s_hor_2_diff_std = FloatField(required=False)
s_hor_2_diff_diff_rel = FloatField(required=False)
## Piston
s_hor_piston_amp = FloatField(required=False)
s_hor_piston_freq = FloatField(required=False)
s_hor_piston_phase = FloatField(required=False)
s_hor_piston_offset = FloatField(required=False)
s_hor_piston_slope = FloatField(required=False)
s_hor_piston_r2 = FloatField(required=False)
s_hor_piston_cycle_min = ListField(FloatField(),required=False)
s_hor_piston_min = FloatField(required=False)
s_hor_piston_min_std = FloatField(required=False)
s_hor_piston_min_diff_rel = FloatField(required=False)
s_hor_piston_cycle_max = ListField(FloatField(),required=False)
s_hor_piston_max = FloatField(required=False)
s_hor_piston_max_std = FloatField(required=False)
s_hor_piston_max_dif_rel = FloatField(required=False)
s_hor_piston_cycle_mean = ListField(FloatField(),required=False)
s_hor_piston_mean = FloatField(required=False)
s_hor_piston_mean_std = FloatField(required=False)
s_hor_piston_mean_diff_rel = FloatField(required=False)
s_hor_piston_cycle_diff = ListField(FloatField(),required=False)
s_hor_piston_diff = FloatField(required=False)
s_hor_piston_diff_std = FloatField(required=False)
s_hor_piston_diff_diff_rel = FloatField(required=False)
s_piston_amp = FloatField(required=False)
s_piston_freq = FloatField(required=False)
s_piston_phase = FloatField(required=False)
s_piston_offset = FloatField(required=False)
s_piston_slope = FloatField(required=False)
s_piston_r2 = FloatField(required=False)
s_piston_cycle_min = ListField(FloatField(),required=False)
s_piston_min = FloatField(required=False)
s_piston_min_std = FloatField(required=False)
s_piston_min_diff_rel = FloatField(required=False)
s_piston_cycle_max = ListField(FloatField(),required=False)
s_piston_max = FloatField(required=False)
s_piston_max_std = FloatField(required=False)
s_piston_max_dif_rel = FloatField(required=False)
s_piston_cycle_mean = ListField(FloatField(),required=False)
s_piston_mean = FloatField(required=False)
s_piston_mean_std = FloatField(required=False)
s_piston_mean_diff_rel = FloatField(required=False)
s_piston_cycle_diff = ListField(FloatField(),required=False)
s_piston_diff = FloatField(required=False)
s_piston_diff_std = FloatField(required=False)
s_piston_diff_diff_rel = FloatField(required=False)

8
src/paveit/datamodels/taskmanager.py
View File

@@ -3,18 +3,20 @@ import datetime
from mongoengine import *
from .infrastructure import MachineBase
from .machines import MachineBase
from .material import Material
from .project import Project
from .usermanagement import Organisation, User
from .workpackage import Workpackage
from .enumeration import ProcessStatusEnum
from .client import Client
class TaskManagerBase(Document):
org_id = LazyReferenceField(Organisation, required=True)
user_id = LazyReferenceField(User, required=True)
client_id = LazyReferenceField(Client, required=True)
project_id = LazyReferenceField(Project, required=True)
wp_id = LazyReferenceField(Workpackage, required=False)
@@ -28,7 +30,7 @@ class TaskManagerBase(Document):
task_finished = DateTimeField(required=False)
assign_users = ListField(LazyReferenceField(User), required=False)
assign_machines = ListField(LazyReferenceField(MachineBase), required=False)
assign_machine = LazyReferenceField(MachineBase, required=False)
series = StringField(default='Serie 01')
@@ -50,6 +52,6 @@ class TaskCITTStiffness(TaskManagerBase):
class TaskDynShearStiffness(TaskManagerBase):
material1 = LazyReferenceField(Material, required=True)
material = LazyReferenceField(Material, required=True)
material2 = LazyReferenceField(Material, required=True)
bounding = LazyReferenceField(Material, required=True)

4
src/paveit/helper/minio.py
View File

@@ -9,7 +9,7 @@ def get_minio_client_processing(bucket_name = 'processing'):
os.environ["MINIO_URL"],
access_key=os.environ["MINIO_ACCESS_KEY"],
secret_key=os.environ["MINIO_SECRET_KEY"],
secure=False
secure=True
)
@@ -28,7 +28,7 @@ def get_minio_client_archive(bucket_name = 'archive'):
os.environ["MINIO_ARCHIVE_URL"],
access_key=os.environ["MINIO_ARCHIVE_ACCESS_KEY"],
secret_key=os.environ["MINIO_ARCHIVE_SECRET_KEY"],
secure=False
secure=True
)
found = client.bucket_exists(bucket_name)

2
src/paveit/helper/mongo.py
View File

@@ -70,7 +70,7 @@ def mongo_get_results(resultsmodel, results: DataFrame, datamodel,
return True
def fetch_recursive(data, fetch_parameter=['norm', 'limits']):
def fetch_recursive(data, fetch_parameter=['norm', 'limits', 'assign_machine']):
fields = data._fields

49
src/paveit/labtest/base.py
View File

@@ -40,12 +40,6 @@ class DataSineLoad():
self._pre_run()
def _which_machine(self):
"""
check the file and try to get the machine from the data
"""
pass
def _set_parameter(self):
self._logger.debug('run _set_parameter')
@@ -57,7 +51,7 @@ class DataSineLoad():
]
self.val_col_names = [
'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2'
'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2', 's_piston'
]
self.columns_analyse = [
@@ -110,7 +104,7 @@ class DataSineLoad():
response.release_conn()
self.data = io.BytesIO(self.data)
self._logger.debug('data read from s3')
self._logger.debug('read data from s3')
def _calc_hash_of_bytesio(self):
self._logger.debug('run _calc_hash_of_bytesio')
@@ -145,6 +139,11 @@ class DataSineLoad():
for key, d in self.metadata.items():
try:
#remove units
for unit in ["°C", 'Hz']:
if unit in d:
d = d.split(unit)[0].strip()
f = float(d.replace(',', '.'))
self.metadata[key] = f
except:
@@ -161,9 +160,27 @@ class DataSineLoad():
self.data.columns = colnames
self._logger.debug(f'columns: {colnames}')
print(self.data.head())
print(self.data['s_piston'].head())
self._logger.debug(f'standardize_data: {self.data.columns}')
def _standardize_meta(self):
self._logger.debug('run _standardize_meta')
# remove "\r\n" ending from Windows
for col in list(self.metadata.keys()):
col_mod = col.replace('\r\n', '')
if col != col_mod:
self.metadata[col_mod] = self.metadata[col]
self.metadata.pop(col)
for par, names in self.meta_names_of_parameter.items():
for name in names:
if name in self.metadata:
@@ -173,6 +190,8 @@ class DataSineLoad():
break
self._logger.debug(f'meta (stand.): {self.metadata}')
def _modify_meta(self):
pass
@@ -184,14 +203,15 @@ class DataSineLoad():
# check if value in metadata:
if name in self.metadata.keys():
self._logger.error(f'add {name} from metadata to data')
self.data[name] = self.metadata[name]
else:
print(name)
self._logger.error(f'{name} not in data')
raise
self._logger.debug(f'validate_data: {self.data.columns}')
def _validate_meta(self):
self._logger.debug('run _validate_meta')
@@ -359,8 +379,7 @@ class DataSineLoad():
self._calc_hash_of_bytesio()
self._define_data_models()
self._data_in_db()
self._which_machine()
#self._data_in_db()
self._set_parameter()
self.update_parameter()
self._define_units()
@@ -371,11 +390,11 @@ class DataSineLoad():
self._process_data()
self._meta_to_float()
self._standardize_data()
self._standardize_meta()
self._standardize_data()
self._modify_meta()
self._validate_data()
self._validate_meta()
self._validate_data()
self._post_string_to_float()
self._post_select_importent_columns()

111
src/paveit/labtest/citt.py
View File

@@ -14,6 +14,45 @@ from paveit.labtest import DataSineLoad
class CITTBase(DataSineLoad):
def _set_parameter(self):
self._logger.debug('run _set_parameter')
self.split_data_based_on_parameter = ['T', 'sigma', 'f']
self.col_as_int = ['N']
self.col_as_float = [
'T', 'F', 's_piston', 's_hor_1', 'f', 's_hor_1', 's_hor_2'
]
self.val_col_names = [
'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2', 's_piston'
]
self.columns_analyse = [
'F', 's_hor_sum', 's_hor_1', 's_hor_2', 's_piston'
]
self.round_values = [('T', 3), ('sigma', 3)]
# Header names after standardization; check if exists
self.val_header_names = ['speciment_height', 'speciment_diameter']
self.number_of_load_cycles_for_analysis = 5
#Dummy Data, replace in Machine Config
self.meta_names_of_parameter = {
'sigma': ['Max. Spannung']
} #list of names
#Dummy Data, replace in Machine Config
self.data_column_names = {
'time': ['Time Series'],
'F': ['Load Series'],
's_hor_1': ['LVDT1 Series'],
's_hor_2': ['LVDT2 Series'],
}
def _sel_df(self, df, num=5, shift=-1):
N = df['N'].unique()
@@ -120,6 +159,8 @@ class CITTBase(DataSineLoad):
try:
self._logger.debug(f'run fit on subset {idx_data}')
data.index = data.index - data.index[0]
res_temp = {}
@@ -200,9 +241,12 @@ class CITTBase(DataSineLoad):
# TODO: Überarbeiten und erweitern (ISSUE #2)
res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[
'fit_s_hor_sum_phase']
except:
except Exception as e:
self._logger.exception(e)
res_temp = None
self._logger.debug(res_temp)
self.fit.append(res_temp)
self.fit = pd.DataFrame.from_records(self.fit)
@@ -267,34 +311,70 @@ class CITTBase(DataSineLoad):
s_hor_sum=list(data['s_hor_sum']),
)
self._logger.debug(f'columns data, {data.columns}')
# add optional datas
for col in ['s_piston']:
if col in data.columns:
self._logger.debug(f'add {col} to output data')
data_out[col] = list(data[col])
outkeys = list(data_out.keys())
self._logger.debug(f'write raw data to db, {outkeys}')
g = CITTSiffness(result=f.id, **data_out).save()
class CITT_TUDresden(CITTBase):
def _which_machine(self):
"""
check the file and try to get the machine from the data
"""
self._machine = 'TIRA'
class CITT_TUDresdenWille(CITTBase):
def _define_units(self):
self.unit_s = 1 / 1000. #mm
self.unit_F = 1.0 #N
self.unit_t = 1. #s
def update_parameter(self):
self.meta_names_of_parameter = {
'sigma': ['Oberspannung'],
'f': ['Prüffrequenz'],
'T': ['Prüftemperatur'],
't': ['Zeit'],
'speciment_diameter': ['Probekörberdurchmesser', 'Probekörberbreite'],
'speciment_height': ['Probekörperhöhe'],
} #list of names
self.data_column_names = {
'time': ['Zeit'],
'F': ['Kraft'],
'N': ['Zyklenzähler'],
's_hor_1': ['Horizontalweg vorn', 'Weg Probe'],
's_hor_2': ['Horizontalweg hinten', 'Weg 2'],
's_piston': ['Kolbenweg'],
}
def _process_data(self):
meta, data = read_geosys(self.data, '039', metadata_ids=['001','003', '011', '019'])
#define in class
self.data = data.reset_index()
self.metadata.update(meta)
# log infos
self._logger.debug(f'metadata: {self.metadata}')
self._logger.debug(f'data: {self.data.head()}')
print(data.head())
class CITT_TUDresdenTira(CITTBase):
def _define_units(self):
if self._machine == 'TIRA':
self.unit_s = 1 #mm
self.unit_F = 1. #N
self.unit_t = 1. #s
def update_parameter(self):
if self._machine == 'TIRA':
self.meta_names_of_parameter = {
'sigma': ['Oberspannung'],
'f': ['Sollfrequnz'],
@@ -302,8 +382,8 @@ class CITT_TUDresden(CITTBase):
#'Nfrom': ['Erster Aufzeichnungslastwechsel', 'Start Cycle'],
#'Nto': ['Letzer Aufzeichnungslastwechsel', 'Last Cycle'],
't': ['Zeit'],
'speciment_diameter': ['PK-Durchmesser'],
'speciment_height': ['PK-Höhe'],
'speciment_diameter': ['PK-Durchmesser', 'Probekörberbreite'],
'speciment_height': ['PK-Höhe', 'Probekörperhöhe'],
} #list of names
self.data_column_names = {
@@ -396,6 +476,7 @@ class CITT_TUDresden(CITTBase):
self._logger.info(self.data.head())
class CITT_PTMDortmund(CITTBase):
def _define_units(self):

10
src/paveit/labtest/sheartest.py
View File

@@ -234,6 +234,8 @@ class ShearTestExtension(ShearTest):
self.fit = []
errors = 0
for idx_data, data in enumerate(self.data):
try:
@@ -326,12 +328,14 @@ class ShearTestExtension(ShearTest):
self.fit.append(res_temp)
if (self.debug) & (len(self.fit) > 5):
if (self.debug) & ( (len(self.fit) > 5) | (errors < 5) ):
break
except Exception as e:
self._logger.critical(e, exc_info=True)
errors += 1
self.fit = pd.DataFrame.from_records(self.fit)
self.fit = self.fit.reset_index(drop=True).set_index('idx')
@@ -449,9 +453,11 @@ class ShearTestExtensionTUDresden(ShearTestExtension):
def update_parameter(self):
self.meta_names_of_parameter = {
't': ['TIME'],
'speciment_diameter': ['Diameter of specimen\r\n'],
'speciment_diameter': ['Diameter of specimen'],
} #list of names
self.data_column_names = {