add more requests

src/paveit/labtest/base.py

@@ -40,12 +40,6 @@ class DataSineLoad():
 
         self._pre_run()
 
-    def _which_machine(self):
-        """
-        check the file and try to get the machine from the data
-        """
-        pass
-
     def _set_parameter(self):
         self._logger.debug('run _set_parameter')
 
@@ -57,7 +51,7 @@ class DataSineLoad():
         ]
 
         self.val_col_names = [
-            'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2'
+            'time', 'T', 'f', 'sigma', 'N', 'F', 's_hor_1', 's_hor_2', 's_piston'
         ]
 
         self.columns_analyse = [
@@ -110,7 +104,7 @@ class DataSineLoad():
             response.release_conn()
 
         self.data = io.BytesIO(self.data)
-        self._logger.debug('data read from s3')
+        self._logger.debug('read data from s3')
 
     def _calc_hash_of_bytesio(self):
         self._logger.debug('run _calc_hash_of_bytesio')
@@ -145,6 +139,11 @@ class DataSineLoad():
 
         for key, d in self.metadata.items():
             try:
+                #remove units
+                for unit in ["°C", 'Hz']:
+                    if unit in d:
+                        d = d.split(unit)[0].strip()
+
                 f = float(d.replace(',', '.'))
                 self.metadata[key] = f
             except:
@@ -160,9 +159,27 @@ class DataSineLoad():
                 colnames = [sub.replace(name, par) for sub in colnames]
 
         self.data.columns = colnames
+        
+        self._logger.debug(f'columns: {colnames}')
+        
+        print(self.data.head())
+        print(self.data['s_piston'].head())
+        
+        self._logger.debug(f'standardize_data: {self.data.columns}')
+        
 
     def _standardize_meta(self):
         self._logger.debug('run _standardize_meta')
+        
+        # remove "\r\n" ending from Windows
+        for col in list(self.metadata.keys()):
+            
+            col_mod = col.replace('\r\n', '')
+            
+            if col != col_mod:
+                self.metadata[col_mod] = self.metadata[col]
+                self.metadata.pop(col)
+            
 
         for par, names in self.meta_names_of_parameter.items():
             for name in names:
@@ -172,6 +189,8 @@ class DataSineLoad():
                     self.metadata.pop(name)
 
                     break
+        
+        self._logger.debug(f'meta (stand.): {self.metadata}')
 
     def _modify_meta(self):
         pass
@@ -184,13 +203,14 @@ class DataSineLoad():
 
                 # check if value in metadata:
                 if name in self.metadata.keys():
-
+                    self._logger.error(f'add {name} from metadata to data')
                     self.data[name] = self.metadata[name]
 
                 else:
-
-                    print(name)
+                    self._logger.error(f'{name} not in data')
                     raise
+        
+        self._logger.debug(f'validate_data: {self.data.columns}')
 
     def _validate_meta(self):
         self._logger.debug('run _validate_meta')
@@ -295,7 +315,7 @@ class DataSineLoad():
 
             gaps = idx_diff > (4 * dt_mean)
             has_gaps = any(gaps)
-
+            
             if has_gaps == False:
                 data_list.append(d)
 
@@ -359,8 +379,7 @@ class DataSineLoad():
 
         self._calc_hash_of_bytesio()
         self._define_data_models()
-        self._data_in_db()
-        self._which_machine()
+        #self._data_in_db()
         self._set_parameter()
         self.update_parameter()
         self._define_units()
@@ -371,11 +390,11 @@ class DataSineLoad():
         self._process_data()
         self._meta_to_float()
 
-        self._standardize_data()
         self._standardize_meta()
+        self._standardize_data()
         self._modify_meta()
-        self._validate_data()
         self._validate_meta()
+        self._validate_data()
 
         self._post_string_to_float()
         self._post_select_importent_columns()