Datenstrukturen und Auswertung angepasst

setup.cfg

@@ -16,6 +16,7 @@ install_requires =
     matplotlib
     seaborn
     mongoengine
+    statsmodels
 
 [options.packages.find]
 where=src

src/paveit/datamodels/__init__.py

@@ -1,7 +1,11 @@
 from .citt import *
 from .data import *
+from .enumeration import *
+from .infrastructure import *
 from .material import *
+from .metrics import *
 from .project import *
 from .sheartest import *
+from .taskmanager import *
 from .usermanagement import *
 from .workpackage import *

src/paveit/datamodels/citt.py

@@ -2,10 +2,7 @@ import datetime
 
 from mongoengine import *
 
-from .material import Material
-from .project import Project
-from .usermanagement import Organisation, User
-from .workpackage import Workpackage
+from .taskmanager import TaskManagerBase
 
 
 class CyclicIndirectTensileTest(Document):
@@ -17,19 +14,16 @@ class CyclicIndirectTensileTest(Document):
 
     standard = StringField(default='TP Asphalt Teil 24')
 
-    org_id = LazyReferenceField(Organisation, required=True)
-    project_id = LazyReferenceField(Project, required=True)
-    workpackage_id = LazyReferenceField(Workpackage, required=False)
+    #org_id = LazyReferenceField(Organisation, required=True)
 
-    user_id = LazyReferenceField(User,
-                                 required=True,
-                                 reverse_delete_rule=DO_NOTHING)
+    #user_id = LazyReferenceField(User,
+    #                             required=True,
+    #                             reverse_delete_rule=DO_NOTHING)
 
-    material = LazyReferenceField(Material, required=True)
+    task_id = LazyReferenceField(TaskManagerBase, required=True)
 
     tags = ListField(StringField())
 
-    machine = StringField(default=None)
 
     filehash = StringField(required=True)
     speciment_name = StringField()
@@ -69,8 +63,23 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
     F_offset = FloatField()
     F_slope = FloatField()
     F_r2 = FloatField()
+    F_cycle_min = ListField(FloatField())
     F_min = FloatField()
+    F_min_std = FloatField()
+    F_min_diff_rel = FloatField()
+    F_cycle_max = ListField(FloatField())
     F_max = FloatField()
+    F_max_std = FloatField()
+    F_max_diff_rel = FloatField()
+    F_cycle_mean = ListField(FloatField())
+    F_mean = FloatField()
+    F_mean_std = FloatField()
+    F_mean_diff_rel = FloatField()
+    F_cycle_diff = ListField(FloatField())
+    F_diff = FloatField()
+    F_diff_std = FloatField()
+    F_diff_diff_rel= FloatField()
+    
     ## S1
     s_hor_1_amp = FloatField()
     s_hor_1_freq = FloatField()
@@ -78,8 +87,22 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
     s_hor_1_offset = FloatField()
     s_hor_1_slope = FloatField()
     s_hor_1_r2 = FloatField()
+    s_hor_1_cycle_min = ListField(FloatField())
     s_hor_1_min = FloatField()
+    s_hor_1_min_std = FloatField()
+    s_hor_1_min_diff_rel = FloatField()
+    s_hor_1_cycle_max = ListField(FloatField())
     s_hor_1_max = FloatField()
+    s_hor_1_max_std = FloatField()
+    s_hor_1_max_diff_rel = FloatField()
+    s_hor_1_cycle_mean = ListField(FloatField())
+    s_hor_1_mean = FloatField()
+    s_hor_1_mean_std = FloatField()
+    s_hor_1_mean_diff_rel = FloatField()
+    s_hor_1_cycle_diff = ListField(FloatField())
+    s_hor_1_diff = FloatField()
+    s_hor_1_diff_std = FloatField()
+    s_hor_1_diff_diff_rel = FloatField()
     ## S2
     s_hor_2_amp = FloatField()
     s_hor_2_freq = FloatField()
@@ -87,8 +110,22 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
     s_hor_2_offset = FloatField()
     s_hor_2_slope = FloatField()
     s_hor_2_r2 = FloatField()
+    s_hor_2_cycle_min = ListField(FloatField())
     s_hor_2_min = FloatField()
+    s_hor_2_min_std = FloatField()
+    s_hor_2_min_diff_rel = FloatField()
+    s_hor_2_cycle_max = ListField(FloatField())
     s_hor_2_max = FloatField()
+    s_hor_2_max_std = FloatField()
+    s_hor_2_max_diff_rel = FloatField()
+    s_hor_2_cycle_mean = ListField(FloatField())
+    s_hor_2_mean = FloatField()
+    s_hor_2_mean_std = FloatField()
+    s_hor_2_mean_diff_rel = FloatField()
+    s_hor_2_cycle_diff = ListField(FloatField())
+    s_hor_2_diff = FloatField()
+    s_hor_2_diff_std = FloatField()
+    s_hor_2_diff_diff_rel = FloatField()
     ## S-Sum
     s_hor_sum_amp = FloatField()
     s_hor_sum_freq = FloatField()
@@ -96,7 +133,44 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
     s_hor_sum_offset = FloatField()
     s_hor_sum_slope = FloatField()
     s_hor_sum_r2 = FloatField()
+    s_hor_sum_cycle_min = ListField(FloatField())
     s_hor_sum_min = FloatField()
+    s_hor_sum_min_std = FloatField()
+    s_hor_sum_min_diff_rel = FloatField()
+    s_hor_sum_cycle_max = ListField(FloatField())
     s_hor_sum_max = FloatField()
+    s_hor_sum_max_std = FloatField()
+    s_hor_sum_max_diff_rel = FloatField()
+    s_hor_sum_cycle_mean = ListField(FloatField())
+    s_hor_sum_mean = FloatField()
+    s_hor_sum_mean_std = FloatField()
+    s_hor_sum_mean_diff_rel = FloatField()
+    s_hor_sum_cycle_diff = ListField(FloatField())
+    s_hor_sum_diff = FloatField()
+    s_hor_sum_diff_std = FloatField()
+    s_hor_sum_diff_diff_rel = FloatField()
 
     #optional parameter
+    ## Piston
+    s_hor_piston_amp = FloatField(required=False)
+    s_hor_piston_freq = FloatField(required=False)
+    s_hor_piston_phase = FloatField(required=False)
+    s_hor_piston_offset = FloatField(required=False)
+    s_hor_piston_slope = FloatField(required=False)
+    s_hor_piston_r2 = FloatField(required=False)
+    s_hor_piston_cycle_min = ListField(FloatField(),required=False)
+    s_hor_piston_min = FloatField(required=False)
+    s_hor_piston_min_std = FloatField(required=False)
+    s_hor_piston_min_diff_rel = FloatField(required=False)
+    s_hor_piston_cycle_max = ListField(FloatField(),required=False)
+    s_hor_piston_max = FloatField(required=False)
+    s_hor_piston_max_std = FloatField(required=False)
+    s_hor_piston_max_dif_rel = FloatField(required=False)
+    s_hor_piston_cycle_mean = ListField(FloatField(),required=False)
+    s_hor_piston_mean = FloatField(required=False)
+    s_hor_piston_mean_std = FloatField(required=False)
+    s_hor_piston_mean_diff_rel = FloatField(required=False)
+    s_hor_piston_cycle_diff = ListField(FloatField(),required=False)
+    s_hor_piston_diff = FloatField(required=False)
+    s_hor_piston_diff_std = FloatField(required=False)
+    s_hor_piston_diff_diff_rel = FloatField(required=False)

src/paveit/datamodels/client.py

@@ -0,0 +1,15 @@
+import datetime
+
+from mongoengine import *
+
+from .usermanagement import Organisation, User
+
+
+class Client(Document):
+    
+    org_id = LazyReferenceField(Organisation, required=True)
+    user_id = LazyReferenceField(User,
+                                 required=True,
+                                 reverse_delete_rule=DO_NOTHING)
+    
+    clientid = StringField(max_length=100)

src/paveit/datamodels/data.py

@@ -13,6 +13,10 @@ class RawData(Document):
                              "step": "60"
                          }})
 
+    time = ListField(FloatField())
+    F = ListField(FloatField())
+    N = ListField(IntField())
+
     meta = {
         'allow_inheritance': True,
         'index_opts': {},
@@ -32,9 +36,6 @@ class DataSheartest(RawData):
                                 reverse_delete_rule=CASCADE)
 
     # data
-    time = ListField(FloatField())
-    F = ListField(FloatField())
-    N = ListField(IntField())
     s_vert_1 = ListField(FloatField())
     s_vert_2 = ListField(FloatField())
     s_piston = ListField(FloatField(), required=False)
@@ -49,9 +50,6 @@ class CITTSiffness(RawData):
                                 reverse_delete_rule=CASCADE)
 
     # data
-    time = ListField(FloatField())
-    F = ListField(FloatField())
-    N = ListField(IntField())
     s_hor_1 = ListField(FloatField())
     s_hor_2 = ListField(FloatField())
     s_hor_sum = ListField(FloatField())

src/paveit/datamodels/enumeration.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class LabtestsEnum(Enum):
+    CITTStiffness = 'CITTStiffness'
+    SHEARStiffness = 'SheartestStiffness'

src/paveit/datamodels/infrastructure.py

@@ -0,0 +1,16 @@
+from mongoengine import *
+
+
+class MachineBase(Document):
+    
+    name = StringField(default=None)
+    
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'infrastructure',
+        "db_alias": 'dblabtests',
+    }

src/paveit/datamodels/material.py

@@ -37,6 +37,7 @@ class Material(Document):
         'index_cls': False,
         'auto_create_index': True,
         'collection': 'materials',
+        "db_alias": 'dblabtests',
         'indexes': [
             [("material", 1)],
             [("name", 1)],

src/paveit/datamodels/metrics.py

@@ -0,0 +1,50 @@
+import datetime
+
+from mongoengine import *
+
+from .project import Project
+from .usermanagement import User, Organisation
+from .workpackage import Workpackage
+
+class MetricsBase(Document):
+
+    date = DateTimeField(default=datetime.datetime.now,
+                         wtf_options={"render_kw": {
+                             "step": "60"
+                         }})
+    
+    app = StringField(required=True, default='backend')
+    
+    method = StringField(required=True)
+    url = StringField(required=False)
+    client = StringField(required=False)
+    
+    status_code = IntField(required=False)
+    
+
+    
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'metrics',
+        "db_alias": 'dblabtests',
+    }
+
+    
+    
+class MetricsBackend(MetricsBase):
+    
+    project_id = LazyReferenceField(Project, required=False)
+    user_id = LazyReferenceField(User,required=False)
+    
+    workpackage_id = LazyReferenceField(Workpackage, required=False)
+    org_id = LazyReferenceField(Organisation, required=False)
+    
+    runtime = FloatField() # in s
+    task = StringField(max_length=30)
+    
+
+    

src/paveit/datamodels/project.py

@@ -2,15 +2,17 @@ import datetime
 
 from mongoengine import *
 
+from .client import Client
 from .usermanagement import Organisation, User
 
 
 class Project(Document):
 
-    name = StringField(required=True)
-    name_short = StringField(required=False)
     project_id = StringField(required=True)
-    client = StringField(required=False)
+    
+    client = LazyReferenceField(Client,
+                                required=True,
+                                reverse_delete_rule=CASCADE)
 
     date = DateTimeField(default=datetime.datetime.now,
                          wtf_options={"render_kw": {
@@ -25,6 +27,10 @@ class Project(Document):
                                  required=False,
                                  reverse_delete_rule=DO_NOTHING)
     
+    name = StringField(required=True)
+    name_short = StringField(required=False)
+    
+
     tags = ListField(StringField())
 
     meta = {
@@ -34,6 +40,7 @@ class Project(Document):
         'index_cls': False,
         'auto_create_index': True,
         'collection': 'projects',
+        "db_alias": 'dblabtests',
         'indexes': [
             [("name_short", 1)],
             [("name", 1)],

src/paveit/datamodels/sheartest.py

@@ -2,10 +2,7 @@ import datetime
 
 from mongoengine import *
 
-from .material import Material
-from .project import Project
-from .usermanagement import Organisation, User
-from .workpackage import Workpackage
+from .taskmanager import TaskManagerBase
 
 
 class DynamicShearTest(Document):
@@ -15,25 +12,13 @@ class DynamicShearTest(Document):
                              "step": "60"
                          }})
     
-    org_id = LazyReferenceField(Organisation, required=True)
-    project_id = LazyReferenceField(Project, required=True)
-    workpackage_id = LazyReferenceField(Workpackage, required=False)
-
-    user_id = LazyReferenceField(User,
-                                 required=True,
-                                 reverse_delete_rule=DO_NOTHING)
-
-    material1 = LazyReferenceField(Material, required=True)
-    material2 = LazyReferenceField(Material, required=True)
-    bounding = LazyReferenceField(Material, required=True)
+    task_id = LazyReferenceField(TaskManagerBase, required=True)
 
     gap_width = FloatField(default=1.0)
 
     tags = ListField(StringField())
 
-    standard = StringField(default='TP Asphalt Teil 24')
-
-    machine = StringField(default=None)
+    standard = StringField(default='TP Asphalt Teil 48 C')
 
     filehash = StringField(required=True)
     speciment_name = StringField()
@@ -79,6 +64,8 @@ class DynamicShearTestExtension(DynamicShearTest):
 
     G = FloatField(required=True)
     broken = BooleanField(required=True)
+    phase = FloatField()
+    
     #fit parameter
     ## required parameters
 
@@ -89,55 +76,117 @@ class DynamicShearTestExtension(DynamicShearTest):
     F_offset = FloatField(required=True)
     F_slope = FloatField(required=True)
     F_r2 = FloatField(required=True)
-    F_max = FloatField(required=True)
-    F_min = FloatField(required=True)
+    F_cycle_min = ListField(FloatField())
+    F_min = FloatField()
+    F_min_std = FloatField()
+    F_min_diff_rel = FloatField()
+    F_cycle_max = ListField(FloatField())
+    F_max = FloatField()
+    F_max_std = FloatField()
+    F_max_diff_rel = FloatField()
+    F_cycle_mean = ListField(FloatField())
+    F_mean = FloatField()
+    F_mean_std = FloatField()
+    F_mean_diff_rel = FloatField()
+    F_cycle_diff = ListField(FloatField())
+    F_diff = FloatField()
+    F_diff_std = FloatField()
+    F_diff_diff_rel= FloatField()
+
     ## S1
-    s_vert_1_amp = FloatField(required=True)
-    s_vert_1_freq = FloatField(required=True)
-    s_vert_1_phase = FloatField(required=True)
-    s_vert_1_offset = FloatField(required=True)
-    s_vert_1_slope = FloatField(required=True)
-    s_vert_1_r2 = FloatField(required=True)
-    s_vert_1_max = FloatField(required=True)
-    s_vert_1_min = FloatField(required=True)
+    s_vert_1_amp = FloatField()
+    s_vert_1_freq = FloatField()
+    s_vert_1_phase = FloatField()
+    s_vert_1_offset = FloatField()
+    s_vert_1_slope = FloatField()
+    s_vert_1_r2 = FloatField()
+    s_vert_1_cycle_min = ListField(FloatField())
+    s_vert_1_min = FloatField()
+    s_vert_1_min_std = FloatField()
+    s_vert_1_min_diff_rel = FloatField()
+    s_vert_1_cycle_max = ListField(FloatField())
+    s_vert_1_max = FloatField()
+    s_vert_1_max_std = FloatField()
+    s_vert_1_max_diff_rel = FloatField()
+    s_vert_1_cycle_mean = ListField(FloatField())
+    s_vert_1_mean = FloatField()
+    s_vert_1_mean_std = FloatField()
+    s_vert_1_mean_diff_rel = FloatField()
+    s_vert_1_cycle_diff = ListField(FloatField())
+    s_vert_1_diff = FloatField()
+    s_vert_1_diff_std = FloatField()
+    s_vert_1_diff_diff_rel = FloatField()
     ## S2
-    s_vert_2_amp = FloatField(required=True)
-    s_vert_2_freq = FloatField(required=True)
-    s_vert_2_phase = FloatField(required=True)
-    s_vert_2_offset = FloatField(required=True)
-    s_vert_2_slope = FloatField(required=True)
-    s_vert_2_r2 = FloatField(required=True)
-    s_vert_2_max = FloatField(required=True)
-    s_vert_2_min = FloatField(required=True)
+    s_vert_2_amp = FloatField()
+    s_vert_2_freq = FloatField()
+    s_vert_2_phase = FloatField()
+    s_vert_2_offset = FloatField()
+    s_vert_2_slope = FloatField()
+    s_vert_2_r2 = FloatField()
+    s_vert_2_cycle_min = ListField(FloatField())
+    s_vert_2_min = FloatField()
+    s_vert_2_min_std = FloatField()
+    s_vert_2_min_diff_rel = FloatField()
+    s_vert_2_cycle_max = ListField(FloatField())
+    s_vert_2_max = FloatField()
+    s_vert_2_max_std = FloatField()
+    s_vert_2_max_diff_rel = FloatField()
+    s_vert_2_cycle_mean = ListField(FloatField())
+    s_vert_2_mean = FloatField()
+    s_vert_2_mean_std = FloatField()
+    s_vert_2_mean_diff_rel = FloatField()
+    s_vert_2_cycle_diff = ListField(FloatField())
+    s_vert_2_diff = FloatField()
+    s_vert_2_diff_std = FloatField()
+    s_vert_2_diff_diff_rel = FloatField()
+    ## S-Sum
+    s_vert_sum_amp = FloatField()
+    s_vert_sum_freq = FloatField()
+    s_vert_sum_phase = FloatField()
+    s_vert_sum_offset = FloatField()
+    s_vert_sum_slope = FloatField()
+    s_vert_sum_r2 = FloatField()
+    s_vert_sum_cycle_min = ListField(FloatField())
+    s_vert_sum_min = FloatField()
+    s_vert_sum_min_std = FloatField()
+    s_vert_sum_min_diff_rel = FloatField()
+    s_vert_sum_cycle_max = ListField(FloatField())
+    s_vert_sum_max = FloatField()
+    s_vert_sum_max_std = FloatField()
+    s_vert_sum_max_diff_rel = FloatField()
+    s_vert_sum_cycle_mean = ListField(FloatField())
+    s_vert_sum_mean = FloatField()
+    s_vert_sum_mean_std = FloatField()
+    s_vert_sum_mean_diff_rel = FloatField()
+    s_vert_sum_cycle_diff = ListField(FloatField())
+    s_vert_sum_diff = FloatField()
+    s_vert_sum_diff_std = FloatField()
+    s_vert_sum_diff_diff_rel = FloatField()
     
     ## optional parameters
-
-    s_vert_sum_amp = FloatField(required=False)
-    s_vert_sum_freq = FloatField(required=False)
-    s_vert_sum_phase = FloatField(required=False)
-    s_vert_sum_offset = FloatField(required=False)
-    s_vert_sum_slope = FloatField(required=False)
-    s_vert_sum_r2 = FloatField(required=False)
-    s_vert_sum_max = FloatField(required=False)
-    s_vert_sum_min = FloatField(required=False)
-
-    s_hor_sum_amp = FloatField(required=False)
-    s_hor_sum_freq = FloatField(required=False)
-    s_hor_sum_phase = FloatField(required=False)
-    s_hor_sum_offset = FloatField(required=False)
-    s_hor_sum_slope = FloatField(required=False)
-    s_hor_sum_r2 = FloatField(required=False)
-    s_hor_sum_max = FloatField(required=False)
-    s_hor_sum_min = FloatField(required=False)
-
+    ## S1
     s_hor_1_amp = FloatField(required=False)
     s_hor_1_freq = FloatField(required=False)
     s_hor_1_phase = FloatField(required=False)
     s_hor_1_offset = FloatField(required=False)
     s_hor_1_slope = FloatField(required=False)
     s_hor_1_r2 = FloatField(required=False)
-    s_hor_1_max = FloatField(required=False)
+    s_hor_1_cycle_min = ListField(FloatField(),required=False)
     s_hor_1_min = FloatField(required=False)
+    s_hor_1_min_std = FloatField(required=False)
+    s_hor_1_min_diff_rel = FloatField(required=False)
+    s_hor_1_cycle_max = ListField(FloatField(),required=False)
+    s_hor_1_max = FloatField(required=False)
+    s_hor_1_max_std = FloatField(required=False)
+    s_hor_1_max_diff_rel = FloatField(required=False)
+    s_hor_1_cycle_mean = ListField(FloatField(),required=False)
+    s_hor_1_mean = FloatField(required=False)
+    s_hor_1_mean_std = FloatField(required=False)
+    s_hor_1_mean_diff_rel = FloatField(required=False)
+    s_hor_1_cycle_diff = ListField(FloatField(),required=False)
+    s_hor_1_diff = FloatField(required=False)
+    s_hor_1_diff_std = FloatField(required=False)
+    s_hor_1_diff_diff_rel = FloatField(required=False)
     ## S2
     s_hor_2_amp = FloatField(required=False)
     s_hor_2_freq = FloatField(required=False)
@@ -145,5 +194,42 @@ class DynamicShearTestExtension(DynamicShearTest):
     s_hor_2_offset = FloatField(required=False)
     s_hor_2_slope = FloatField(required=False)
     s_hor_2_r2 = FloatField(required=False)
-    s_hor_2_max = FloatField(required=False)
+    s_hor_2_cycle_min = ListField(FloatField(),required=False)
     s_hor_2_min = FloatField(required=False)
+    s_hor_2_min_std = FloatField(required=False)
+    s_hor_2_min_diff_rel = FloatField(required=False)
+    s_hor_2_cycle_max = ListField(FloatField(),required=False)
+    s_hor_2_max = FloatField(required=False)
+    s_hor_2_max_std = FloatField(required=False)
+    s_hor_2_max_diff_rel = FloatField(required=False)
+    s_hor_2_cycle_mean = ListField(FloatField(), required=False)
+    s_hor_2_mean = FloatField(required=False)
+    s_hor_2_mean_std = FloatField(required=False)
+    s_hor_2_mean_diff_rel = FloatField(required=False)
+    s_hor_2_cycle_diff = ListField(FloatField(), required=False)
+    s_hor_2_diff = FloatField(required=False)
+    s_hor_2_diff_std = FloatField(required=False)
+    s_hor_2_diff_diff_rel = FloatField(required=False)
+    ## Piston
+    s_hor_piston_amp = FloatField(required=False)
+    s_hor_piston_freq = FloatField(required=False)
+    s_hor_piston_phase = FloatField(required=False)
+    s_hor_piston_offset = FloatField(required=False)
+    s_hor_piston_slope = FloatField(required=False)
+    s_hor_piston_r2 = FloatField(required=False)
+    s_hor_piston_cycle_min = ListField(FloatField(),required=False)
+    s_hor_piston_min = FloatField(required=False)
+    s_hor_piston_min_std = FloatField(required=False)
+    s_hor_piston_min_diff_rel = FloatField(required=False)
+    s_hor_piston_cycle_max = ListField(FloatField(),required=False)
+    s_hor_piston_max = FloatField(required=False)
+    s_hor_piston_max_std = FloatField(required=False)
+    s_hor_piston_max_dif_rel = FloatField(required=False)
+    s_hor_piston_cycle_mean = ListField(FloatField(),required=False)
+    s_hor_piston_mean = FloatField(required=False)
+    s_hor_piston_mean_std = FloatField(required=False)
+    s_hor_piston_mean_diff_rel = FloatField(required=False)
+    s_hor_piston_cycle_diff = ListField(FloatField(),required=False)
+    s_hor_piston_diff = FloatField(required=False)
+    s_hor_piston_diff_std = FloatField(required=False)
+    s_hor_piston_diff_diff_rel = FloatField(required=False)

src/paveit/datamodels/taskmanager.py

@@ -0,0 +1,54 @@
+
+import datetime
+
+from mongoengine import *
+
+from .infrastructure import MachineBase
+from .material import Material
+from .project import Project
+from .usermanagement import Organisation, User
+from .workpackage import Workpackage
+
+
+class TaskManagerBase(Document):
+    
+    org_id = LazyReferenceField(Organisation, required=True)
+    user_id = LazyReferenceField(User, required=True)
+    project_id = LazyReferenceField(Project, required=True)
+    wp_id = LazyReferenceField(Workpackage, required=False)
+        
+    finieshed = BooleanField(default=False)
+    
+    task_added = DateTimeField(default=datetime.datetime.now,
+                               wtf_options={"r  ender_kw": {
+                                 "step": "60"
+                                 }})
+    
+    task_finished = DateTimeField(required=False)
+    
+    assign_users = ListField(LazyReferenceField(User), required=False)
+    assign_machines = ListField(LazyReferenceField(MachineBase), required=False)
+    
+    series = StringField(default='Serie 01')
+    
+    meta = {
+        'allow_inheritance': True,
+        'index_opts': {},
+        'index_background': True,
+        'index_cls': False,
+        'auto_create_index': True,
+        'collection': 'taskmanager',
+        "db_alias": 'dblabtests',
+    }
+    
+    
+class TaskCITTStiffness(TaskManagerBase):
+    
+    material = LazyReferenceField(Material, required=True)
+    
+    
+class TaskDynShearStiffness(TaskManagerBase):
+    
+    material1 = LazyReferenceField(Material, required=True)
+    material2 = LazyReferenceField(Material, required=True)
+    bounding = LazyReferenceField(Material, required=True)

src/paveit/datamodels/usermanagement.py

@@ -23,7 +23,7 @@ class Organisation(Document):
         'index_cls': False,
         'auto_create_index': True,
         'collection': 'organisation',
-        'db_alias': 'dbusers',
+        'db_alias': 'dblabtests',
     }
 
 
@@ -40,7 +40,11 @@ class User(Document):
                                    "step": "60"
                                }})
 
-    name = StringField(required=True)
+    name = StringField(required=False)
+    
+    email = EmailField(required=True)
+    
+    password = StringField(required=True)
 
     meta = {
         'allow_inheritance': True,
@@ -48,5 +52,6 @@ class User(Document):
         'index_background': True,
         'index_cls': False,
         'auto_create_index': True,
-        'collection': 'user'
+        'collection': 'user',
+        'db_alias': 'dblabtests',
     }

src/paveit/datamodels/workpackage.py

@@ -29,5 +29,6 @@ class Workpackage(Document):
         'index_background': True,
         'index_cls': False,
         'auto_create_index': True,
-        'collection': 'workpackages'
+        'collection': 'workpackages',
+        "db_alias": 'dblabtests',
     }

src/paveit/labtest/base.py

@@ -119,6 +119,19 @@ class DataSineLoad():
         self.data.seek(0)
         self._logger.debug(f'Hash of file: {self.filehash}')
         
+    def _define_data_models(self):
+        pass
+        
+    def _data_in_db(self):
+        
+        nsamples = self._datamodel.objects(filehash = self.filehash).count()
+        
+        if nsamples>0:
+            self.file_in_db = True
+        else:
+            self.file_in_db = False
+        
+
     def _process_data(self):
         """ convert self.data (BytesIO) to pandas.DataFrame, update
         self.metadata with informations from file """
@@ -194,8 +207,11 @@ class DataSineLoad():
             return
 
         for col in sel.columns:
-            self.data[col] = pd.to_numeric(self.data[col].str.replace(
+            try:
+                self.data[col] = pd.to_numeric(self.data[col].str.replace(
                 ',', '.'))
+            except:
+                pass
 
     def _post_apply_units(self):
 
@@ -342,6 +358,8 @@ class DataSineLoad():
             self._read_from_s3_to_bytesio()
 
         self._calc_hash_of_bytesio()
+        self._define_data_models()
+        self._data_in_db()
         self._which_machine()
         self._set_parameter()
         self.update_parameter()

src/paveit/labtest/citt.py

@@ -5,6 +5,7 @@ from csv import reader
 import numpy as np
 import pandas as pd
 from bson import ObjectId
+
 from paveit import calc_nu, fit_cos
 from paveit.datamodels import CITTSiffness, CITTSiffnessResults
 from paveit.io import read_geosys
@@ -135,14 +136,43 @@ class CITTBase(DataSineLoad):
                     if not col in data.columns: continue
 
                     y = data[col].values
-
                     res = fit_cos(x, y, freq=freq)
 
                     for key, value in res.items():
                         res_temp[f'fit_{col}_{key}'] = value
                         
-                    res_temp[f'fit_{col}_max'] = max(y)
-                    res_temp[f'fit_{col}_min'] = min(y)
+
+                    # analyse cycle data
+                    
+                    cycle_min = []
+                    cycle_max = []
+                    cycle_mean = []
+                    cycle_diff = []
+                    
+                    for N, data_cycle in data.groupby('N'):
+                        y = data_cycle[col].values
+                        
+                        cycle_min.append(y.min())
+                        cycle_max.append(y.max())
+                        cycle_mean.append(y.mean())
+                        cycle_diff.append(cycle_max[-1] - cycle_min[-1])
+                        
+                    res_temp[f'fit_{col}_cycle_min'] = cycle_min
+                    res_temp[f'fit_{col}_min'] = np.mean(cycle_min)
+                    res_temp[f'fit_{col}_min_std'] = np.std(cycle_min)
+                    res_temp[f'fit_{col}_min_diff_rel'] = (np.max(cycle_min) - np.min(cycle_min))/np.mean(cycle_min)
+                    res_temp[f'fit_{col}_cycle_max'] = cycle_max
+                    res_temp[f'fit_{col}_max'] = np.mean(cycle_max)
+                    res_temp[f'fit_{col}_max_std'] = np.std(cycle_max)
+                    res_temp[f'fit_{col}_max_diff_rel'] = (np.max(cycle_max) - np.min(cycle_max))/np.mean(cycle_max)
+                    res_temp[f'fit_{col}_cycle_mean'] = cycle_mean
+                    res_temp[f'fit_{col}_mean'] = np.mean(cycle_mean)
+                    res_temp[f'fit_{col}_mean_std'] = np.std(cycle_mean)
+                    res_temp[f'fit_{col}_mean_diff_rel'] = (np.max(cycle_mean) - np.min(cycle_mean))/np.mean(cycle_mean)
+                    res_temp[f'fit_{col}_cycle_diff'] = cycle_diff
+                    res_temp[f'fit_{col}_diff'] = np.mean(cycle_diff)
+                    res_temp[f'fit_{col}_diff_std'] = np.std(cycle_diff)
+                    res_temp[f'fit_{col}_diff_diff_rel'] = (np.max(cycle_diff) - np.min(cycle_diff))/np.mean(cycle_diff)
 
                 # add more metadata
                 res_temp['f_set'] = freq
@@ -186,12 +216,9 @@ class CITTBase(DataSineLoad):
         self._logger.debug(self.fit['stiffness'])
 
     def save(self,
-             org_id: ObjectId,
-             project_id: ObjectId,
-             material_id: ObjectId,
-             user_id: ObjectId,
-             meta: dict = {},
-             wp_id: ObjectId | None = None):
+             task_id: ObjectId,
+             meta: dict = {}
+             ):
         """
         save results to mongodb
         """
@@ -200,16 +227,13 @@ class CITTBase(DataSineLoad):
             raise
 
         # precheck data and results
-        assert len(self.data) == len(self.fit)
+        #assert len(self.data) == len(self.fit)
 
         for idx_fit, fit in self.fit.iterrows():
             data = self.data[idx_fit]
 
             meta['filehash'] = self.filehash
-            meta['org_id'] = org_id
-            meta['project_id'] = project_id
-            meta['material'] = material_id
-            meta['user_id'] = user_id
+            meta['task_id'] = task_id
 
             #check if result in db
             #n = CITTSiffness.objects(**meta).count()
@@ -726,11 +750,13 @@ class CITT_LaborHart(CITTBase):
         data.columns = head
 
         # FIX: Sigma nicht in Metadaten oder Messdaten enthalten
-        sigma = float(
-            os.path.split(self.filename)[-1].split('MPa')[0].strip().replace(
-                ',', '.'))
+        print(meta)
+        if not "sigma" in self.metadata:
+            sigma = float(
+                os.path.split(self.filename)[-1].split('MPa')[0].strip().replace(
+                    ',', '.'))
 
-        meta['sigma'] = sigma
+            meta['sigma'] = sigma
 
         #clean data
         data = data.dropna(axis=1)
@@ -847,3 +873,100 @@ class CITT_BAGKoeln(CITTBase):
         # log infos
         self._logger.info(self.metadata)
         self._logger.info(self.data.head())
+
+class CITT_Braunschweig(CITTBase):
+
+    def _define_units(self):
+
+        self.unit_s = 1.0  #mm
+        self.unit_F = -1000.0  #N
+        self.unit_t = 1.0
+
+    def update_parameter(self):
+
+        self.meta_names_of_parameter = {
+            't': ['t [s]'],
+        }  #list of names
+
+        self.data_column_names = {
+            'time': ['t [s]'],
+            'f': [ 'FREQUENZ_x10'],
+            'F': [ 'F act [kN]'],
+            's_hor_1': ['r 1A [mm]'],
+            's_hor_2': ['r 1B [mm]'],
+            's_piston': ['s piston [mm]'],
+        }
+
+    def _process_data(self):
+        
+        self._logger.info(self.metadata)
+
+        meta = {}
+
+        encoding = 'latin-1'
+
+        self.data.seek(0)
+
+        head = pd.read_csv(self.data,
+                           skiprows=14,
+                           sep=';',
+                           nrows=1,
+                           encoding=encoding,
+                           decimal=',',
+                           thousands='.',
+                           skip_blank_lines=True).columns
+
+        head = [h.strip() for h in head]
+
+        self.data.seek(0)
+        data = pd.read_csv(self.data,
+                           skiprows=20,
+                           sep=';',
+                           encoding=encoding,
+                           decimal=',',
+                           thousands='.',
+                           header=None,
+                           skip_blank_lines=True)
+
+        data.columns = head
+
+        # add frequency
+        ## [start, ncycles]
+        sel = {0.1: [0, 11], 1.0: [11, 20], 5.0: [30, 100], 10: [130, 111]}
+
+        N_f_dict = {}
+
+        Ns = 0
+        for freq in sel.keys():
+
+            par = sel[freq]
+
+            N = par[0]
+            dN = par[1]
+
+            for i in range(N, N + dN):
+
+                N_f_dict[Ns] = freq
+
+                Ns += 1
+
+        data['f'] = data['N'].replace(N_f_dict)
+        data = data[data['f'] <= 10.0]
+
+        #sigma
+        data['sigma'] = self.metadata['sigma']
+        data['T'] = self.metadata['temperature']
+
+        #clean data
+        #data = data.dropna(axis=0)
+        #data = data.dropna(axis=1)
+        
+        data = data.drop(columns=['Date Time'])
+
+        #define in class
+        self.data = data
+        self.metadata.update(meta)
+
+        # log infos
+        self._logger.info(self.metadata)
+        self._logger.info(self.data.head())

src/paveit/labtest/sheartest.py

@@ -6,6 +6,7 @@ import lmfit as lm
 import numpy as np
 import pandas as pd
 from bson import ObjectId
+
 from paveit import calc_nu, fit_cos
 from paveit.datamodels import DataSheartest, DynamicShearTestExtension
 from paveit.io import read_geosys
@@ -108,18 +109,19 @@ class ShearTest(DataSineLoad):
 
 class ShearTestExtension(ShearTest):
     
+    def _define_data_models(self):
+        
+        self._datamodel = DynamicShearTestExtension
+        self._datamodel_raw = DataSheartest
+
     def save(
             self,
             org_id: ObjectId,
-            project_id: ObjectId,
-            material_1_id: ObjectId,
-            material_2_id: ObjectId,
-            material_boundary_id: ObjectId,
             user_id: ObjectId,
+            task_id: ObjectId,
+            broken: bool,
+            gap_width: float,  #mm
             meta: dict = {},
-            wp_id: ObjectId | None = None,
-            broken: bool = False,
-            gap_width: float = 1.0,  #mm
     ):
         """
         save results to mongodb
@@ -134,19 +136,10 @@ class ShearTestExtension(ShearTest):
         for idx_fit, fit in self.fit.iterrows():
             data = self.data[idx_fit]
             
+            meta['task_id'] = org_id
+            meta['gap_width'] = gap_width
             meta['filehash'] = self.filehash
-            meta['org_id'] = org_id
-            meta['project_id'] = project_id
-            meta['workpackage_id'] = wp_id
-            meta['user_id'] = user_id
-
-            meta['material1'] = material_1_id
-            meta['material2'] = material_2_id
-            meta['bounding'] = material_boundary_id
-
-            #check if result in db
-            #n = CITTSiffness.objects(**meta).count()
-            #print(n)
+            meta['speciment_name'] = self.filename
 
             # write data
             data_dict = fit.to_dict()
@@ -164,7 +157,7 @@ class ShearTestExtension(ShearTest):
                 d[new] = d[old]
                 d.pop(old)
 
-            f = DynamicShearTestExtension(**data_dict).save()
+            f = self._datamodel (**data_dict).save()
 
             # required data
             data_out = dict(
@@ -180,7 +173,7 @@ class ShearTestExtension(ShearTest):
                 if col in data.columns:
                     data_out[col] = list(data[col])
 
-            g = DataSheartest(result=f.id, **data_out).save()
+            g = self._datamodel_raw(result=f.id, **data_out).save()
 
     def _fit_select_data(self):
         """ 
@@ -261,8 +254,38 @@ class ShearTestExtension(ShearTest):
                     for key, value in res.items():
                         res_temp[f'fit_{col}_{key}'] = value
 
-                    res_temp[f'fit_{col}_max'] = max(y)
-                    res_temp[f'fit_{col}_min'] = min(y)
+                    # analyse cycle data
+                    
+                    cycle_min = []
+                    cycle_max = []
+                    cycle_mean = []
+                    cycle_diff = []
+                    
+                    for N, data_cycle in data.groupby('N'):
+                        y = data_cycle[col].values
+                        
+                        cycle_min.append(y.min())
+                        cycle_max.append(y.max())
+                        cycle_mean.append(y.mean())
+                        cycle_diff.append(cycle_max[-1] - cycle_min[-1])
+                        
+                    res_temp[f'fit_{col}_cycle_min'] = cycle_min
+                    res_temp[f'fit_{col}_min'] = np.mean(cycle_min)
+                    res_temp[f'fit_{col}_min_std'] = np.std(cycle_min)
+                    res_temp[f'fit_{col}_min_diff_rel'] = abs((np.max(cycle_min) - np.min(cycle_min))/np.mean(cycle_min))
+                    res_temp[f'fit_{col}_cycle_max'] = cycle_max
+                    res_temp[f'fit_{col}_max'] = np.mean(cycle_max)
+                    res_temp[f'fit_{col}_max_std'] = np.std(cycle_max)
+                    res_temp[f'fit_{col}_max_diff_rel'] = abs((np.max(cycle_max) - np.min(cycle_max))/np.mean(cycle_max))
+                    res_temp[f'fit_{col}_cycle_mean'] = cycle_mean
+                    res_temp[f'fit_{col}_mean'] = np.mean(cycle_mean)
+                    res_temp[f'fit_{col}_mean_std'] = np.std(cycle_mean)
+                    res_temp[f'fit_{col}_mean_diff_rel'] = abs((np.max(cycle_mean) - np.min(cycle_mean))/np.mean(cycle_mean))
+                    res_temp[f'fit_{col}_cycle_diff'] = cycle_diff
+                    res_temp[f'fit_{col}_diff'] = np.mean(cycle_diff)
+                    res_temp[f'fit_{col}_diff_std'] = np.std(cycle_diff)
+                    res_temp[f'fit_{col}_diff_diff_rel'] = abs((np.max(cycle_diff) - np.min(cycle_diff))/np.mean(cycle_diff))
+
 
                 # add more metadata
                 res_temp['f_set'] = freq
@@ -288,9 +311,9 @@ class ShearTestExtension(ShearTest):
 
                 res_temp['G'] = tau / gamma
 
-                #metadaten
-                #for c in ['T', 'extension', 'sigma_normal', 'f']:
-                #    res_temp[c] = res_temp[c][0]
+                # TODO: Überarbeiten und erweitern (ISSUE #2)
+                res_temp['phase'] = res_temp['fit_F_phase'] - res_temp[
+                    'fit_s_vert_sum_phase']
 
                 self.fit.append(res_temp)