From 471fa8dabf6496d574ad11f4dc1eaddcb2275f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Clau=C3=9F?= Date: Fri, 17 Mar 2023 14:54:19 +0100 Subject: [PATCH] Datenstrukturen und Auswertung angepasst --- Makefile | 0 README.md | 0 debug.csv | 0 setup.cfg | 1 + setup.py | 0 src/paveit/__init__.py | 0 src/paveit/analysis/__init__.py | 0 src/paveit/analysis/regression.py | 0 src/paveit/datamodels/__init__.py | 4 + src/paveit/datamodels/citt.py | 100 ++++++++-- src/paveit/datamodels/client.py | 15 ++ src/paveit/datamodels/data.py | 10 +- src/paveit/datamodels/enumeration.py | 6 + src/paveit/datamodels/infrastructure.py | 16 ++ src/paveit/datamodels/material.py | 1 + src/paveit/datamodels/metrics.py | 50 +++++ src/paveit/datamodels/project.py | 13 +- src/paveit/datamodels/sheartest.py | 204 ++++++++++++++------ src/paveit/datamodels/taskmanager.py | 54 ++++++ src/paveit/datamodels/usermanagement.py | 11 +- src/paveit/datamodels/workpackage.py | 3 +- src/paveit/functions/__init__.py | 0 src/paveit/functions/citt.py | 0 src/paveit/helper/__init__.py | 0 src/paveit/helper/filehandling.py | 0 src/paveit/helper/filehasher.py | 0 src/paveit/helper/minio.py | 0 src/paveit/helper/mongo.py | 0 src/paveit/io/__init__.py | 0 src/paveit/io/geosys.py | 0 src/paveit/labtest/__init__.py | 0 src/paveit/labtest/base.py | 20 +- src/paveit/labtest/citt.py | 159 +++++++++++++-- src/paveit/labtest/sheartest.py | 77 +++++--- tests/__init__.py | 0 tests/analysis/__init__.py | 0 tests/analysis/citt_test.py | 0 tests/analysis/sine_test.py | 0 tests/data/citt/PTM_Dortmund/meta.toml | 0 tests/data/citt/PTM_Dortmund/sample_01.xlsm | Bin tests/helper/__init__.py | 0 tests/helper/filehandling_test.py | 0 42 files changed, 613 insertions(+), 131 deletions(-) mode change 100644 => 100755 Makefile mode change 100644 => 100755 README.md mode change 100644 => 100755 debug.csv mode change 100644 => 100755 setup.cfg mode change 100644 => 100755 setup.py mode change 100644 => 100755 src/paveit/__init__.py mode change 100644 => 100755 src/paveit/analysis/__init__.py mode change 100644 => 100755 src/paveit/analysis/regression.py mode change 100644 => 100755 src/paveit/datamodels/__init__.py mode change 100644 => 100755 src/paveit/datamodels/citt.py create mode 100644 src/paveit/datamodels/client.py mode change 100644 => 100755 src/paveit/datamodels/data.py create mode 100644 src/paveit/datamodels/enumeration.py create mode 100644 src/paveit/datamodels/infrastructure.py mode change 100644 => 100755 src/paveit/datamodels/material.py create mode 100644 src/paveit/datamodels/metrics.py mode change 100644 => 100755 src/paveit/datamodels/project.py mode change 100644 => 100755 src/paveit/datamodels/sheartest.py create mode 100644 src/paveit/datamodels/taskmanager.py mode change 100644 => 100755 src/paveit/datamodels/usermanagement.py mode change 100644 => 100755 src/paveit/datamodels/workpackage.py mode change 100644 => 100755 src/paveit/functions/__init__.py mode change 100644 => 100755 src/paveit/functions/citt.py mode change 100644 => 100755 src/paveit/helper/__init__.py mode change 100644 => 100755 src/paveit/helper/filehandling.py mode change 100644 => 100755 src/paveit/helper/filehasher.py mode change 100644 => 100755 src/paveit/helper/minio.py mode change 100644 => 100755 src/paveit/helper/mongo.py mode change 100644 => 100755 src/paveit/io/__init__.py mode change 100644 => 100755 src/paveit/io/geosys.py mode change 100644 => 100755 src/paveit/labtest/__init__.py mode change 100644 => 100755 src/paveit/labtest/base.py mode change 100644 => 100755 src/paveit/labtest/citt.py mode change 100644 => 100755 src/paveit/labtest/sheartest.py mode change 100644 => 100755 tests/__init__.py mode change 100644 => 100755 tests/analysis/__init__.py mode change 100644 => 100755 tests/analysis/citt_test.py mode change 100644 => 100755 tests/analysis/sine_test.py mode change 100644 => 100755 tests/data/citt/PTM_Dortmund/meta.toml mode change 100644 => 100755 tests/data/citt/PTM_Dortmund/sample_01.xlsm mode change 100644 => 100755 tests/helper/__init__.py mode change 100644 => 100755 tests/helper/filehandling_test.py diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/debug.csv b/debug.csv old mode 100644 new mode 100755 diff --git a/setup.cfg b/setup.cfg old mode 100644 new mode 100755 index 30e2d44..937a161 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,7 @@ install_requires = matplotlib seaborn mongoengine + statsmodels [options.packages.find] where=src diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 diff --git a/src/paveit/__init__.py b/src/paveit/__init__.py old mode 100644 new mode 100755 diff --git a/src/paveit/analysis/__init__.py b/src/paveit/analysis/__init__.py old mode 100644 new mode 100755 diff --git a/src/paveit/analysis/regression.py b/src/paveit/analysis/regression.py old mode 100644 new mode 100755 diff --git a/src/paveit/datamodels/__init__.py b/src/paveit/datamodels/__init__.py old mode 100644 new mode 100755 index 0f62da9..ffacc4d --- a/src/paveit/datamodels/__init__.py +++ b/src/paveit/datamodels/__init__.py @@ -1,7 +1,11 @@ from .citt import * from .data import * +from .enumeration import * +from .infrastructure import * from .material import * +from .metrics import * from .project import * from .sheartest import * +from .taskmanager import * from .usermanagement import * from .workpackage import * diff --git a/src/paveit/datamodels/citt.py b/src/paveit/datamodels/citt.py old mode 100644 new mode 100755 index 1c31186..48442a0 --- a/src/paveit/datamodels/citt.py +++ b/src/paveit/datamodels/citt.py @@ -2,10 +2,7 @@ import datetime from mongoengine import * -from .material import Material -from .project import Project -from .usermanagement import Organisation, User -from .workpackage import Workpackage +from .taskmanager import TaskManagerBase class CyclicIndirectTensileTest(Document): @@ -17,19 +14,16 @@ class CyclicIndirectTensileTest(Document): standard = StringField(default='TP Asphalt Teil 24') - org_id = LazyReferenceField(Organisation, required=True) - project_id = LazyReferenceField(Project, required=True) - workpackage_id = LazyReferenceField(Workpackage, required=False) + #org_id = LazyReferenceField(Organisation, required=True) - user_id = LazyReferenceField(User, - required=True, - reverse_delete_rule=DO_NOTHING) + #user_id = LazyReferenceField(User, + # required=True, + # reverse_delete_rule=DO_NOTHING) - material = LazyReferenceField(Material, required=True) + task_id = LazyReferenceField(TaskManagerBase, required=True) tags = ListField(StringField()) - machine = StringField(default=None) filehash = StringField(required=True) speciment_name = StringField() @@ -69,8 +63,23 @@ class CITTSiffnessResults(CyclicIndirectTensileTest): F_offset = FloatField() F_slope = FloatField() F_r2 = FloatField() + F_cycle_min = ListField(FloatField()) F_min = FloatField() + F_min_std = FloatField() + F_min_diff_rel = FloatField() + F_cycle_max = ListField(FloatField()) F_max = FloatField() + F_max_std = FloatField() + F_max_diff_rel = FloatField() + F_cycle_mean = ListField(FloatField()) + F_mean = FloatField() + F_mean_std = FloatField() + F_mean_diff_rel = FloatField() + F_cycle_diff = ListField(FloatField()) + F_diff = FloatField() + F_diff_std = FloatField() + F_diff_diff_rel= FloatField() + ## S1 s_hor_1_amp = FloatField() s_hor_1_freq = FloatField() @@ -78,8 +87,22 @@ class CITTSiffnessResults(CyclicIndirectTensileTest): s_hor_1_offset = FloatField() s_hor_1_slope = FloatField() s_hor_1_r2 = FloatField() + s_hor_1_cycle_min = ListField(FloatField()) s_hor_1_min = FloatField() + s_hor_1_min_std = FloatField() + s_hor_1_min_diff_rel = FloatField() + s_hor_1_cycle_max = ListField(FloatField()) s_hor_1_max = FloatField() + s_hor_1_max_std = FloatField() + s_hor_1_max_diff_rel = FloatField() + s_hor_1_cycle_mean = ListField(FloatField()) + s_hor_1_mean = FloatField() + s_hor_1_mean_std = FloatField() + s_hor_1_mean_diff_rel = FloatField() + s_hor_1_cycle_diff = ListField(FloatField()) + s_hor_1_diff = FloatField() + s_hor_1_diff_std = FloatField() + s_hor_1_diff_diff_rel = FloatField() ## S2 s_hor_2_amp = FloatField() s_hor_2_freq = FloatField() @@ -87,8 +110,22 @@ class CITTSiffnessResults(CyclicIndirectTensileTest): s_hor_2_offset = FloatField() s_hor_2_slope = FloatField() s_hor_2_r2 = FloatField() + s_hor_2_cycle_min = ListField(FloatField()) s_hor_2_min = FloatField() + s_hor_2_min_std = FloatField() + s_hor_2_min_diff_rel = FloatField() + s_hor_2_cycle_max = ListField(FloatField()) s_hor_2_max = FloatField() + s_hor_2_max_std = FloatField() + s_hor_2_max_diff_rel = FloatField() + s_hor_2_cycle_mean = ListField(FloatField()) + s_hor_2_mean = FloatField() + s_hor_2_mean_std = FloatField() + s_hor_2_mean_diff_rel = FloatField() + s_hor_2_cycle_diff = ListField(FloatField()) + s_hor_2_diff = FloatField() + s_hor_2_diff_std = FloatField() + s_hor_2_diff_diff_rel = FloatField() ## S-Sum s_hor_sum_amp = FloatField() s_hor_sum_freq = FloatField() @@ -96,7 +133,44 @@ class CITTSiffnessResults(CyclicIndirectTensileTest): s_hor_sum_offset = FloatField() s_hor_sum_slope = FloatField() s_hor_sum_r2 = FloatField() + s_hor_sum_cycle_min = ListField(FloatField()) s_hor_sum_min = FloatField() + s_hor_sum_min_std = FloatField() + s_hor_sum_min_diff_rel = FloatField() + s_hor_sum_cycle_max = ListField(FloatField()) s_hor_sum_max = FloatField() + s_hor_sum_max_std = FloatField() + s_hor_sum_max_diff_rel = FloatField() + s_hor_sum_cycle_mean = ListField(FloatField()) + s_hor_sum_mean = FloatField() + s_hor_sum_mean_std = FloatField() + s_hor_sum_mean_diff_rel = FloatField() + s_hor_sum_cycle_diff = ListField(FloatField()) + s_hor_sum_diff = FloatField() + s_hor_sum_diff_std = FloatField() + s_hor_sum_diff_diff_rel = FloatField() - #optional parameter \ No newline at end of file + #optional parameter + ## Piston + s_hor_piston_amp = FloatField(required=False) + s_hor_piston_freq = FloatField(required=False) + s_hor_piston_phase = FloatField(required=False) + s_hor_piston_offset = FloatField(required=False) + s_hor_piston_slope = FloatField(required=False) + s_hor_piston_r2 = FloatField(required=False) + s_hor_piston_cycle_min = ListField(FloatField(),required=False) + s_hor_piston_min = FloatField(required=False) + s_hor_piston_min_std = FloatField(required=False) + s_hor_piston_min_diff_rel = FloatField(required=False) + s_hor_piston_cycle_max = ListField(FloatField(),required=False) + s_hor_piston_max = FloatField(required=False) + s_hor_piston_max_std = FloatField(required=False) + s_hor_piston_max_dif_rel = FloatField(required=False) + s_hor_piston_cycle_mean = ListField(FloatField(),required=False) + s_hor_piston_mean = FloatField(required=False) + s_hor_piston_mean_std = FloatField(required=False) + s_hor_piston_mean_diff_rel = FloatField(required=False) + s_hor_piston_cycle_diff = ListField(FloatField(),required=False) + s_hor_piston_diff = FloatField(required=False) + s_hor_piston_diff_std = FloatField(required=False) + s_hor_piston_diff_diff_rel = FloatField(required=False) \ No newline at end of file diff --git a/src/paveit/datamodels/client.py b/src/paveit/datamodels/client.py new file mode 100644 index 0000000..9928af2 --- /dev/null +++ b/src/paveit/datamodels/client.py @@ -0,0 +1,15 @@ +import datetime + +from mongoengine import * + +from .usermanagement import Organisation, User + + +class Client(Document): + + org_id = LazyReferenceField(Organisation, required=True) + user_id = LazyReferenceField(User, + required=True, + reverse_delete_rule=DO_NOTHING) + + clientid = StringField(max_length=100) \ No newline at end of file diff --git a/src/paveit/datamodels/data.py b/src/paveit/datamodels/data.py old mode 100644 new mode 100755 index 8ee302a..2292707 --- a/src/paveit/datamodels/data.py +++ b/src/paveit/datamodels/data.py @@ -13,6 +13,10 @@ class RawData(Document): "step": "60" }}) + time = ListField(FloatField()) + F = ListField(FloatField()) + N = ListField(IntField()) + meta = { 'allow_inheritance': True, 'index_opts': {}, @@ -32,9 +36,6 @@ class DataSheartest(RawData): reverse_delete_rule=CASCADE) # data - time = ListField(FloatField()) - F = ListField(FloatField()) - N = ListField(IntField()) s_vert_1 = ListField(FloatField()) s_vert_2 = ListField(FloatField()) s_piston = ListField(FloatField(), required=False) @@ -49,9 +50,6 @@ class CITTSiffness(RawData): reverse_delete_rule=CASCADE) # data - time = ListField(FloatField()) - F = ListField(FloatField()) - N = ListField(IntField()) s_hor_1 = ListField(FloatField()) s_hor_2 = ListField(FloatField()) s_hor_sum = ListField(FloatField()) diff --git a/src/paveit/datamodels/enumeration.py b/src/paveit/datamodels/enumeration.py new file mode 100644 index 0000000..73ac892 --- /dev/null +++ b/src/paveit/datamodels/enumeration.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class LabtestsEnum(Enum): + CITTStiffness = 'CITTStiffness' + SHEARStiffness = 'SheartestStiffness' \ No newline at end of file diff --git a/src/paveit/datamodels/infrastructure.py b/src/paveit/datamodels/infrastructure.py new file mode 100644 index 0000000..e02ff8c --- /dev/null +++ b/src/paveit/datamodels/infrastructure.py @@ -0,0 +1,16 @@ +from mongoengine import * + + +class MachineBase(Document): + + name = StringField(default=None) + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'infrastructure', + "db_alias": 'dblabtests', + } \ No newline at end of file diff --git a/src/paveit/datamodels/material.py b/src/paveit/datamodels/material.py old mode 100644 new mode 100755 index 98c75df..4a9e1ec --- a/src/paveit/datamodels/material.py +++ b/src/paveit/datamodels/material.py @@ -37,6 +37,7 @@ class Material(Document): 'index_cls': False, 'auto_create_index': True, 'collection': 'materials', + "db_alias": 'dblabtests', 'indexes': [ [("material", 1)], [("name", 1)], diff --git a/src/paveit/datamodels/metrics.py b/src/paveit/datamodels/metrics.py new file mode 100644 index 0000000..bba6d7b --- /dev/null +++ b/src/paveit/datamodels/metrics.py @@ -0,0 +1,50 @@ +import datetime + +from mongoengine import * + +from .project import Project +from .usermanagement import User, Organisation +from .workpackage import Workpackage + +class MetricsBase(Document): + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + app = StringField(required=True, default='backend') + + method = StringField(required=True) + url = StringField(required=False) + client = StringField(required=False) + + status_code = IntField(required=False) + + + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'metrics', + "db_alias": 'dblabtests', + } + + + +class MetricsBackend(MetricsBase): + + project_id = LazyReferenceField(Project, required=False) + user_id = LazyReferenceField(User,required=False) + + workpackage_id = LazyReferenceField(Workpackage, required=False) + org_id = LazyReferenceField(Organisation, required=False) + + runtime = FloatField() # in s + task = StringField(max_length=30) + + + \ No newline at end of file diff --git a/src/paveit/datamodels/project.py b/src/paveit/datamodels/project.py old mode 100644 new mode 100755 index fe82cc2..5fdd03e --- a/src/paveit/datamodels/project.py +++ b/src/paveit/datamodels/project.py @@ -2,15 +2,17 @@ import datetime from mongoengine import * +from .client import Client from .usermanagement import Organisation, User class Project(Document): - name = StringField(required=True) - name_short = StringField(required=False) project_id = StringField(required=True) - client = StringField(required=False) + + client = LazyReferenceField(Client, + required=True, + reverse_delete_rule=CASCADE) date = DateTimeField(default=datetime.datetime.now, wtf_options={"render_kw": { @@ -24,6 +26,10 @@ class Project(Document): user_id = LazyReferenceField(User, required=False, reverse_delete_rule=DO_NOTHING) + + name = StringField(required=True) + name_short = StringField(required=False) + tags = ListField(StringField()) @@ -34,6 +40,7 @@ class Project(Document): 'index_cls': False, 'auto_create_index': True, 'collection': 'projects', + "db_alias": 'dblabtests', 'indexes': [ [("name_short", 1)], [("name", 1)], diff --git a/src/paveit/datamodels/sheartest.py b/src/paveit/datamodels/sheartest.py old mode 100644 new mode 100755 index 92b3ee0..5e9b074 --- a/src/paveit/datamodels/sheartest.py +++ b/src/paveit/datamodels/sheartest.py @@ -2,10 +2,7 @@ import datetime from mongoengine import * -from .material import Material -from .project import Project -from .usermanagement import Organisation, User -from .workpackage import Workpackage +from .taskmanager import TaskManagerBase class DynamicShearTest(Document): @@ -14,26 +11,14 @@ class DynamicShearTest(Document): wtf_options={"render_kw": { "step": "60" }}) - - org_id = LazyReferenceField(Organisation, required=True) - project_id = LazyReferenceField(Project, required=True) - workpackage_id = LazyReferenceField(Workpackage, required=False) - - user_id = LazyReferenceField(User, - required=True, - reverse_delete_rule=DO_NOTHING) - - material1 = LazyReferenceField(Material, required=True) - material2 = LazyReferenceField(Material, required=True) - bounding = LazyReferenceField(Material, required=True) + + task_id = LazyReferenceField(TaskManagerBase, required=True) gap_width = FloatField(default=1.0) tags = ListField(StringField()) - standard = StringField(default='TP Asphalt Teil 24') - - machine = StringField(default=None) + standard = StringField(default='TP Asphalt Teil 48 C') filehash = StringField(required=True) speciment_name = StringField() @@ -79,6 +64,8 @@ class DynamicShearTestExtension(DynamicShearTest): G = FloatField(required=True) broken = BooleanField(required=True) + phase = FloatField() + #fit parameter ## required parameters @@ -89,55 +76,117 @@ class DynamicShearTestExtension(DynamicShearTest): F_offset = FloatField(required=True) F_slope = FloatField(required=True) F_r2 = FloatField(required=True) - F_max = FloatField(required=True) - F_min = FloatField(required=True) + F_cycle_min = ListField(FloatField()) + F_min = FloatField() + F_min_std = FloatField() + F_min_diff_rel = FloatField() + F_cycle_max = ListField(FloatField()) + F_max = FloatField() + F_max_std = FloatField() + F_max_diff_rel = FloatField() + F_cycle_mean = ListField(FloatField()) + F_mean = FloatField() + F_mean_std = FloatField() + F_mean_diff_rel = FloatField() + F_cycle_diff = ListField(FloatField()) + F_diff = FloatField() + F_diff_std = FloatField() + F_diff_diff_rel= FloatField() + ## S1 - s_vert_1_amp = FloatField(required=True) - s_vert_1_freq = FloatField(required=True) - s_vert_1_phase = FloatField(required=True) - s_vert_1_offset = FloatField(required=True) - s_vert_1_slope = FloatField(required=True) - s_vert_1_r2 = FloatField(required=True) - s_vert_1_max = FloatField(required=True) - s_vert_1_min = FloatField(required=True) + s_vert_1_amp = FloatField() + s_vert_1_freq = FloatField() + s_vert_1_phase = FloatField() + s_vert_1_offset = FloatField() + s_vert_1_slope = FloatField() + s_vert_1_r2 = FloatField() + s_vert_1_cycle_min = ListField(FloatField()) + s_vert_1_min = FloatField() + s_vert_1_min_std = FloatField() + s_vert_1_min_diff_rel = FloatField() + s_vert_1_cycle_max = ListField(FloatField()) + s_vert_1_max = FloatField() + s_vert_1_max_std = FloatField() + s_vert_1_max_diff_rel = FloatField() + s_vert_1_cycle_mean = ListField(FloatField()) + s_vert_1_mean = FloatField() + s_vert_1_mean_std = FloatField() + s_vert_1_mean_diff_rel = FloatField() + s_vert_1_cycle_diff = ListField(FloatField()) + s_vert_1_diff = FloatField() + s_vert_1_diff_std = FloatField() + s_vert_1_diff_diff_rel = FloatField() ## S2 - s_vert_2_amp = FloatField(required=True) - s_vert_2_freq = FloatField(required=True) - s_vert_2_phase = FloatField(required=True) - s_vert_2_offset = FloatField(required=True) - s_vert_2_slope = FloatField(required=True) - s_vert_2_r2 = FloatField(required=True) - s_vert_2_max = FloatField(required=True) - s_vert_2_min = FloatField(required=True) - + s_vert_2_amp = FloatField() + s_vert_2_freq = FloatField() + s_vert_2_phase = FloatField() + s_vert_2_offset = FloatField() + s_vert_2_slope = FloatField() + s_vert_2_r2 = FloatField() + s_vert_2_cycle_min = ListField(FloatField()) + s_vert_2_min = FloatField() + s_vert_2_min_std = FloatField() + s_vert_2_min_diff_rel = FloatField() + s_vert_2_cycle_max = ListField(FloatField()) + s_vert_2_max = FloatField() + s_vert_2_max_std = FloatField() + s_vert_2_max_diff_rel = FloatField() + s_vert_2_cycle_mean = ListField(FloatField()) + s_vert_2_mean = FloatField() + s_vert_2_mean_std = FloatField() + s_vert_2_mean_diff_rel = FloatField() + s_vert_2_cycle_diff = ListField(FloatField()) + s_vert_2_diff = FloatField() + s_vert_2_diff_std = FloatField() + s_vert_2_diff_diff_rel = FloatField() + ## S-Sum + s_vert_sum_amp = FloatField() + s_vert_sum_freq = FloatField() + s_vert_sum_phase = FloatField() + s_vert_sum_offset = FloatField() + s_vert_sum_slope = FloatField() + s_vert_sum_r2 = FloatField() + s_vert_sum_cycle_min = ListField(FloatField()) + s_vert_sum_min = FloatField() + s_vert_sum_min_std = FloatField() + s_vert_sum_min_diff_rel = FloatField() + s_vert_sum_cycle_max = ListField(FloatField()) + s_vert_sum_max = FloatField() + s_vert_sum_max_std = FloatField() + s_vert_sum_max_diff_rel = FloatField() + s_vert_sum_cycle_mean = ListField(FloatField()) + s_vert_sum_mean = FloatField() + s_vert_sum_mean_std = FloatField() + s_vert_sum_mean_diff_rel = FloatField() + s_vert_sum_cycle_diff = ListField(FloatField()) + s_vert_sum_diff = FloatField() + s_vert_sum_diff_std = FloatField() + s_vert_sum_diff_diff_rel = FloatField() + ## optional parameters - - s_vert_sum_amp = FloatField(required=False) - s_vert_sum_freq = FloatField(required=False) - s_vert_sum_phase = FloatField(required=False) - s_vert_sum_offset = FloatField(required=False) - s_vert_sum_slope = FloatField(required=False) - s_vert_sum_r2 = FloatField(required=False) - s_vert_sum_max = FloatField(required=False) - s_vert_sum_min = FloatField(required=False) - - s_hor_sum_amp = FloatField(required=False) - s_hor_sum_freq = FloatField(required=False) - s_hor_sum_phase = FloatField(required=False) - s_hor_sum_offset = FloatField(required=False) - s_hor_sum_slope = FloatField(required=False) - s_hor_sum_r2 = FloatField(required=False) - s_hor_sum_max = FloatField(required=False) - s_hor_sum_min = FloatField(required=False) - + ## S1 s_hor_1_amp = FloatField(required=False) s_hor_1_freq = FloatField(required=False) s_hor_1_phase = FloatField(required=False) s_hor_1_offset = FloatField(required=False) s_hor_1_slope = FloatField(required=False) s_hor_1_r2 = FloatField(required=False) - s_hor_1_max = FloatField(required=False) + s_hor_1_cycle_min = ListField(FloatField(),required=False) s_hor_1_min = FloatField(required=False) + s_hor_1_min_std = FloatField(required=False) + s_hor_1_min_diff_rel = FloatField(required=False) + s_hor_1_cycle_max = ListField(FloatField(),required=False) + s_hor_1_max = FloatField(required=False) + s_hor_1_max_std = FloatField(required=False) + s_hor_1_max_diff_rel = FloatField(required=False) + s_hor_1_cycle_mean = ListField(FloatField(),required=False) + s_hor_1_mean = FloatField(required=False) + s_hor_1_mean_std = FloatField(required=False) + s_hor_1_mean_diff_rel = FloatField(required=False) + s_hor_1_cycle_diff = ListField(FloatField(),required=False) + s_hor_1_diff = FloatField(required=False) + s_hor_1_diff_std = FloatField(required=False) + s_hor_1_diff_diff_rel = FloatField(required=False) ## S2 s_hor_2_amp = FloatField(required=False) s_hor_2_freq = FloatField(required=False) @@ -145,5 +194,42 @@ class DynamicShearTestExtension(DynamicShearTest): s_hor_2_offset = FloatField(required=False) s_hor_2_slope = FloatField(required=False) s_hor_2_r2 = FloatField(required=False) + s_hor_2_cycle_min = ListField(FloatField(),required=False) + s_hor_2_min = FloatField(required=False) + s_hor_2_min_std = FloatField(required=False) + s_hor_2_min_diff_rel = FloatField(required=False) + s_hor_2_cycle_max = ListField(FloatField(),required=False) s_hor_2_max = FloatField(required=False) - s_hor_2_min = FloatField(required=False) \ No newline at end of file + s_hor_2_max_std = FloatField(required=False) + s_hor_2_max_diff_rel = FloatField(required=False) + s_hor_2_cycle_mean = ListField(FloatField(), required=False) + s_hor_2_mean = FloatField(required=False) + s_hor_2_mean_std = FloatField(required=False) + s_hor_2_mean_diff_rel = FloatField(required=False) + s_hor_2_cycle_diff = ListField(FloatField(), required=False) + s_hor_2_diff = FloatField(required=False) + s_hor_2_diff_std = FloatField(required=False) + s_hor_2_diff_diff_rel = FloatField(required=False) + ## Piston + s_hor_piston_amp = FloatField(required=False) + s_hor_piston_freq = FloatField(required=False) + s_hor_piston_phase = FloatField(required=False) + s_hor_piston_offset = FloatField(required=False) + s_hor_piston_slope = FloatField(required=False) + s_hor_piston_r2 = FloatField(required=False) + s_hor_piston_cycle_min = ListField(FloatField(),required=False) + s_hor_piston_min = FloatField(required=False) + s_hor_piston_min_std = FloatField(required=False) + s_hor_piston_min_diff_rel = FloatField(required=False) + s_hor_piston_cycle_max = ListField(FloatField(),required=False) + s_hor_piston_max = FloatField(required=False) + s_hor_piston_max_std = FloatField(required=False) + s_hor_piston_max_dif_rel = FloatField(required=False) + s_hor_piston_cycle_mean = ListField(FloatField(),required=False) + s_hor_piston_mean = FloatField(required=False) + s_hor_piston_mean_std = FloatField(required=False) + s_hor_piston_mean_diff_rel = FloatField(required=False) + s_hor_piston_cycle_diff = ListField(FloatField(),required=False) + s_hor_piston_diff = FloatField(required=False) + s_hor_piston_diff_std = FloatField(required=False) + s_hor_piston_diff_diff_rel = FloatField(required=False) \ No newline at end of file diff --git a/src/paveit/datamodels/taskmanager.py b/src/paveit/datamodels/taskmanager.py new file mode 100644 index 0000000..470632a --- /dev/null +++ b/src/paveit/datamodels/taskmanager.py @@ -0,0 +1,54 @@ + +import datetime + +from mongoengine import * + +from .infrastructure import MachineBase +from .material import Material +from .project import Project +from .usermanagement import Organisation, User +from .workpackage import Workpackage + + +class TaskManagerBase(Document): + + org_id = LazyReferenceField(Organisation, required=True) + user_id = LazyReferenceField(User, required=True) + project_id = LazyReferenceField(Project, required=True) + wp_id = LazyReferenceField(Workpackage, required=False) + + finieshed = BooleanField(default=False) + + task_added = DateTimeField(default=datetime.datetime.now, + wtf_options={"r ender_kw": { + "step": "60" + }}) + + task_finished = DateTimeField(required=False) + + assign_users = ListField(LazyReferenceField(User), required=False) + assign_machines = ListField(LazyReferenceField(MachineBase), required=False) + + series = StringField(default='Serie 01') + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'taskmanager', + "db_alias": 'dblabtests', + } + + +class TaskCITTStiffness(TaskManagerBase): + + material = LazyReferenceField(Material, required=True) + + +class TaskDynShearStiffness(TaskManagerBase): + + material1 = LazyReferenceField(Material, required=True) + material2 = LazyReferenceField(Material, required=True) + bounding = LazyReferenceField(Material, required=True) \ No newline at end of file diff --git a/src/paveit/datamodels/usermanagement.py b/src/paveit/datamodels/usermanagement.py old mode 100644 new mode 100755 index 75e8769..d43a0e3 --- a/src/paveit/datamodels/usermanagement.py +++ b/src/paveit/datamodels/usermanagement.py @@ -23,7 +23,7 @@ class Organisation(Document): 'index_cls': False, 'auto_create_index': True, 'collection': 'organisation', - 'db_alias': 'dbusers', + 'db_alias': 'dblabtests', } @@ -40,7 +40,11 @@ class User(Document): "step": "60" }}) - name = StringField(required=True) + name = StringField(required=False) + + email = EmailField(required=True) + + password = StringField(required=True) meta = { 'allow_inheritance': True, @@ -48,5 +52,6 @@ class User(Document): 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'collection': 'user' + 'collection': 'user', + 'db_alias': 'dblabtests', } \ No newline at end of file diff --git a/src/paveit/datamodels/workpackage.py b/src/paveit/datamodels/workpackage.py old mode 100644 new mode 100755 index 16040b6..8a04ee3 --- a/src/paveit/datamodels/workpackage.py +++ b/src/paveit/datamodels/workpackage.py @@ -29,5 +29,6 @@ class Workpackage(Document): 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'collection': 'workpackages' + 'collection': 'workpackages', + "db_alias": 'dblabtests', } \ No newline at end of file diff --git a/src/paveit/functions/__init__.py b/src/paveit/functions/__init__.py old mode 100644 new mode 100755 diff --git a/src/paveit/functions/citt.py b/src/paveit/functions/citt.py old mode 100644 new mode 100755 diff --git a/src/paveit/helper/__init__.py b/src/paveit/helper/__init__.py old mode 100644 new mode 100755 diff --git a/src/paveit/helper/filehandling.py b/src/paveit/helper/filehandling.py old mode 100644 new mode 100755 diff --git a/src/paveit/helper/filehasher.py b/src/paveit/helper/filehasher.py old mode 100644 new mode 100755 diff --git a/src/paveit/helper/minio.py b/src/paveit/helper/minio.py old mode 100644 new mode 100755 diff --git a/src/paveit/helper/mongo.py b/src/paveit/helper/mongo.py old mode 100644 new mode 100755 diff --git a/src/paveit/io/__init__.py b/src/paveit/io/__init__.py old mode 100644 new mode 100755 diff --git a/src/paveit/io/geosys.py b/src/paveit/io/geosys.py old mode 100644 new mode 100755 diff --git a/src/paveit/labtest/__init__.py b/src/paveit/labtest/__init__.py old mode 100644 new mode 100755 diff --git a/src/paveit/labtest/base.py b/src/paveit/labtest/base.py old mode 100644 new mode 100755 index 1dc49a3..d7b905d --- a/src/paveit/labtest/base.py +++ b/src/paveit/labtest/base.py @@ -118,6 +118,19 @@ class DataSineLoad(): self.filehash = calc_hash_of_bytes(self.data) self.data.seek(0) self._logger.debug(f'Hash of file: {self.filehash}') + + def _define_data_models(self): + pass + + def _data_in_db(self): + + nsamples = self._datamodel.objects(filehash = self.filehash).count() + + if nsamples>0: + self.file_in_db = True + else: + self.file_in_db = False + def _process_data(self): """ convert self.data (BytesIO) to pandas.DataFrame, update @@ -194,8 +207,11 @@ class DataSineLoad(): return for col in sel.columns: - self.data[col] = pd.to_numeric(self.data[col].str.replace( + try: + self.data[col] = pd.to_numeric(self.data[col].str.replace( ',', '.')) + except: + pass def _post_apply_units(self): @@ -342,6 +358,8 @@ class DataSineLoad(): self._read_from_s3_to_bytesio() self._calc_hash_of_bytesio() + self._define_data_models() + self._data_in_db() self._which_machine() self._set_parameter() self.update_parameter() diff --git a/src/paveit/labtest/citt.py b/src/paveit/labtest/citt.py old mode 100644 new mode 100755 index 31fd963..6293bd1 --- a/src/paveit/labtest/citt.py +++ b/src/paveit/labtest/citt.py @@ -5,6 +5,7 @@ from csv import reader import numpy as np import pandas as pd from bson import ObjectId + from paveit import calc_nu, fit_cos from paveit.datamodels import CITTSiffness, CITTSiffnessResults from paveit.io import read_geosys @@ -135,14 +136,43 @@ class CITTBase(DataSineLoad): if not col in data.columns: continue y = data[col].values - res = fit_cos(x, y, freq=freq) for key, value in res.items(): res_temp[f'fit_{col}_{key}'] = value + - res_temp[f'fit_{col}_max'] = max(y) - res_temp[f'fit_{col}_min'] = min(y) + # analyse cycle data + + cycle_min = [] + cycle_max = [] + cycle_mean = [] + cycle_diff = [] + + for N, data_cycle in data.groupby('N'): + y = data_cycle[col].values + + cycle_min.append(y.min()) + cycle_max.append(y.max()) + cycle_mean.append(y.mean()) + cycle_diff.append(cycle_max[-1] - cycle_min[-1]) + + res_temp[f'fit_{col}_cycle_min'] = cycle_min + res_temp[f'fit_{col}_min'] = np.mean(cycle_min) + res_temp[f'fit_{col}_min_std'] = np.std(cycle_min) + res_temp[f'fit_{col}_min_diff_rel'] = (np.max(cycle_min) - np.min(cycle_min))/np.mean(cycle_min) + res_temp[f'fit_{col}_cycle_max'] = cycle_max + res_temp[f'fit_{col}_max'] = np.mean(cycle_max) + res_temp[f'fit_{col}_max_std'] = np.std(cycle_max) + res_temp[f'fit_{col}_max_diff_rel'] = (np.max(cycle_max) - np.min(cycle_max))/np.mean(cycle_max) + res_temp[f'fit_{col}_cycle_mean'] = cycle_mean + res_temp[f'fit_{col}_mean'] = np.mean(cycle_mean) + res_temp[f'fit_{col}_mean_std'] = np.std(cycle_mean) + res_temp[f'fit_{col}_mean_diff_rel'] = (np.max(cycle_mean) - np.min(cycle_mean))/np.mean(cycle_mean) + res_temp[f'fit_{col}_cycle_diff'] = cycle_diff + res_temp[f'fit_{col}_diff'] = np.mean(cycle_diff) + res_temp[f'fit_{col}_diff_std'] = np.std(cycle_diff) + res_temp[f'fit_{col}_diff_diff_rel'] = (np.max(cycle_diff) - np.min(cycle_diff))/np.mean(cycle_diff) # add more metadata res_temp['f_set'] = freq @@ -186,12 +216,9 @@ class CITTBase(DataSineLoad): self._logger.debug(self.fit['stiffness']) def save(self, - org_id: ObjectId, - project_id: ObjectId, - material_id: ObjectId, - user_id: ObjectId, - meta: dict = {}, - wp_id: ObjectId | None = None): + task_id: ObjectId, + meta: dict = {} + ): """ save results to mongodb """ @@ -200,16 +227,13 @@ class CITTBase(DataSineLoad): raise # precheck data and results - assert len(self.data) == len(self.fit) + #assert len(self.data) == len(self.fit) for idx_fit, fit in self.fit.iterrows(): data = self.data[idx_fit] meta['filehash'] = self.filehash - meta['org_id'] = org_id - meta['project_id'] = project_id - meta['material'] = material_id - meta['user_id'] = user_id + meta['task_id'] = task_id #check if result in db #n = CITTSiffness.objects(**meta).count() @@ -726,11 +750,13 @@ class CITT_LaborHart(CITTBase): data.columns = head # FIX: Sigma nicht in Metadaten oder Messdaten enthalten - sigma = float( - os.path.split(self.filename)[-1].split('MPa')[0].strip().replace( - ',', '.')) + print(meta) + if not "sigma" in self.metadata: + sigma = float( + os.path.split(self.filename)[-1].split('MPa')[0].strip().replace( + ',', '.')) - meta['sigma'] = sigma + meta['sigma'] = sigma #clean data data = data.dropna(axis=1) @@ -847,3 +873,100 @@ class CITT_BAGKoeln(CITTBase): # log infos self._logger.info(self.metadata) self._logger.info(self.data.head()) + +class CITT_Braunschweig(CITTBase): + + def _define_units(self): + + self.unit_s = 1.0 #mm + self.unit_F = -1000.0 #N + self.unit_t = 1.0 + + def update_parameter(self): + + self.meta_names_of_parameter = { + 't': ['t [s]'], + } #list of names + + self.data_column_names = { + 'time': ['t [s]'], + 'f': [ 'FREQUENZ_x10'], + 'F': [ 'F act [kN]'], + 's_hor_1': ['r 1A [mm]'], + 's_hor_2': ['r 1B [mm]'], + 's_piston': ['s piston [mm]'], + } + + def _process_data(self): + + self._logger.info(self.metadata) + + meta = {} + + encoding = 'latin-1' + + self.data.seek(0) + + head = pd.read_csv(self.data, + skiprows=14, + sep=';', + nrows=1, + encoding=encoding, + decimal=',', + thousands='.', + skip_blank_lines=True).columns + + head = [h.strip() for h in head] + + self.data.seek(0) + data = pd.read_csv(self.data, + skiprows=20, + sep=';', + encoding=encoding, + decimal=',', + thousands='.', + header=None, + skip_blank_lines=True) + + data.columns = head + + # add frequency + ## [start, ncycles] + sel = {0.1: [0, 11], 1.0: [11, 20], 5.0: [30, 100], 10: [130, 111]} + + N_f_dict = {} + + Ns = 0 + for freq in sel.keys(): + + par = sel[freq] + + N = par[0] + dN = par[1] + + for i in range(N, N + dN): + + N_f_dict[Ns] = freq + + Ns += 1 + + data['f'] = data['N'].replace(N_f_dict) + data = data[data['f'] <= 10.0] + + #sigma + data['sigma'] = self.metadata['sigma'] + data['T'] = self.metadata['temperature'] + + #clean data + #data = data.dropna(axis=0) + #data = data.dropna(axis=1) + + data = data.drop(columns=['Date Time']) + + #define in class + self.data = data + self.metadata.update(meta) + + # log infos + self._logger.info(self.metadata) + self._logger.info(self.data.head()) diff --git a/src/paveit/labtest/sheartest.py b/src/paveit/labtest/sheartest.py old mode 100644 new mode 100755 index 039d889..fc902bc --- a/src/paveit/labtest/sheartest.py +++ b/src/paveit/labtest/sheartest.py @@ -6,6 +6,7 @@ import lmfit as lm import numpy as np import pandas as pd from bson import ObjectId + from paveit import calc_nu, fit_cos from paveit.datamodels import DataSheartest, DynamicShearTestExtension from paveit.io import read_geosys @@ -107,19 +108,20 @@ class ShearTest(DataSineLoad): class ShearTestExtension(ShearTest): + + def _define_data_models(self): + + self._datamodel = DynamicShearTestExtension + self._datamodel_raw = DataSheartest def save( self, org_id: ObjectId, - project_id: ObjectId, - material_1_id: ObjectId, - material_2_id: ObjectId, - material_boundary_id: ObjectId, user_id: ObjectId, + task_id: ObjectId, + broken: bool, + gap_width: float, #mm meta: dict = {}, - wp_id: ObjectId | None = None, - broken: bool = False, - gap_width: float = 1.0, #mm ): """ save results to mongodb @@ -133,20 +135,11 @@ class ShearTestExtension(ShearTest): for idx_fit, fit in self.fit.iterrows(): data = self.data[idx_fit] - + + meta['task_id'] = org_id + meta['gap_width'] = gap_width meta['filehash'] = self.filehash - meta['org_id'] = org_id - meta['project_id'] = project_id - meta['workpackage_id'] = wp_id - meta['user_id'] = user_id - - meta['material1'] = material_1_id - meta['material2'] = material_2_id - meta['bounding'] = material_boundary_id - - #check if result in db - #n = CITTSiffness.objects(**meta).count() - #print(n) + meta['speciment_name'] = self.filename # write data data_dict = fit.to_dict() @@ -164,7 +157,7 @@ class ShearTestExtension(ShearTest): d[new] = d[old] d.pop(old) - f = DynamicShearTestExtension(**data_dict).save() + f = self._datamodel (**data_dict).save() # required data data_out = dict( @@ -180,7 +173,7 @@ class ShearTestExtension(ShearTest): if col in data.columns: data_out[col] = list(data[col]) - g = DataSheartest(result=f.id, **data_out).save() + g = self._datamodel_raw(result=f.id, **data_out).save() def _fit_select_data(self): """ @@ -261,8 +254,38 @@ class ShearTestExtension(ShearTest): for key, value in res.items(): res_temp[f'fit_{col}_{key}'] = value - res_temp[f'fit_{col}_max'] = max(y) - res_temp[f'fit_{col}_min'] = min(y) + # analyse cycle data + + cycle_min = [] + cycle_max = [] + cycle_mean = [] + cycle_diff = [] + + for N, data_cycle in data.groupby('N'): + y = data_cycle[col].values + + cycle_min.append(y.min()) + cycle_max.append(y.max()) + cycle_mean.append(y.mean()) + cycle_diff.append(cycle_max[-1] - cycle_min[-1]) + + res_temp[f'fit_{col}_cycle_min'] = cycle_min + res_temp[f'fit_{col}_min'] = np.mean(cycle_min) + res_temp[f'fit_{col}_min_std'] = np.std(cycle_min) + res_temp[f'fit_{col}_min_diff_rel'] = abs((np.max(cycle_min) - np.min(cycle_min))/np.mean(cycle_min)) + res_temp[f'fit_{col}_cycle_max'] = cycle_max + res_temp[f'fit_{col}_max'] = np.mean(cycle_max) + res_temp[f'fit_{col}_max_std'] = np.std(cycle_max) + res_temp[f'fit_{col}_max_diff_rel'] = abs((np.max(cycle_max) - np.min(cycle_max))/np.mean(cycle_max)) + res_temp[f'fit_{col}_cycle_mean'] = cycle_mean + res_temp[f'fit_{col}_mean'] = np.mean(cycle_mean) + res_temp[f'fit_{col}_mean_std'] = np.std(cycle_mean) + res_temp[f'fit_{col}_mean_diff_rel'] = abs((np.max(cycle_mean) - np.min(cycle_mean))/np.mean(cycle_mean)) + res_temp[f'fit_{col}_cycle_diff'] = cycle_diff + res_temp[f'fit_{col}_diff'] = np.mean(cycle_diff) + res_temp[f'fit_{col}_diff_std'] = np.std(cycle_diff) + res_temp[f'fit_{col}_diff_diff_rel'] = abs((np.max(cycle_diff) - np.min(cycle_diff))/np.mean(cycle_diff)) + # add more metadata res_temp['f_set'] = freq @@ -288,9 +311,9 @@ class ShearTestExtension(ShearTest): res_temp['G'] = tau / gamma - #metadaten - #for c in ['T', 'extension', 'sigma_normal', 'f']: - # res_temp[c] = res_temp[c][0] + # TODO: Überarbeiten und erweitern (ISSUE #2) + res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[ + 'fit_s_vert_sum_phase'] self.fit.append(res_temp) diff --git a/tests/__init__.py b/tests/__init__.py old mode 100644 new mode 100755 diff --git a/tests/analysis/__init__.py b/tests/analysis/__init__.py old mode 100644 new mode 100755 diff --git a/tests/analysis/citt_test.py b/tests/analysis/citt_test.py old mode 100644 new mode 100755 diff --git a/tests/analysis/sine_test.py b/tests/analysis/sine_test.py old mode 100644 new mode 100755 diff --git a/tests/data/citt/PTM_Dortmund/meta.toml b/tests/data/citt/PTM_Dortmund/meta.toml old mode 100644 new mode 100755 diff --git a/tests/data/citt/PTM_Dortmund/sample_01.xlsm b/tests/data/citt/PTM_Dortmund/sample_01.xlsm old mode 100644 new mode 100755 diff --git a/tests/helper/__init__.py b/tests/helper/__init__.py old mode 100644 new mode 100755 diff --git a/tests/helper/filehandling_test.py b/tests/helper/filehandling_test.py old mode 100644 new mode 100755