refactor(generator): draft filter usage

Author: Joao

pixels/generator/filters.py

@@ -25,10 +25,9 @@ def _make_mask_on_value(img, mask_value):
     return mask_img
 
 
-def order_tensor_on_masks(images, mask_value, max_images=12):
+def order_tensor_on_masks(images: np.array, mask_value: float, max_images: int = 12):
     """
     Order a set of images based on a mask count.
-
     Parameters
     ----------
         images : array
@@ -37,7 +36,6 @@ def order_tensor_on_masks(images, mask_value, max_images=12):
             Value to create mask.
         max_images : int
             The maximum number of images to return
-
     Returns
     -------
         image : numpy array

pixels/generator/generator.py

@@ -411,7 +411,6 @@ def get_data(self, index, metadata=False):
         if temp_dir:
             temp_dir.cleanup()
         x_imgs = np.array(x_imgs)
-
         if (
             SENTINEL_2 in self.platforms or LANDSAT_8 in self.platforms
         ) and self.cloud_sort:
@@ -420,7 +419,6 @@ def get_data(self, index, metadata=False):
             x_imgs = filters.order_tensor_on_cloud_mask(
                 x_imgs, max_images=self.timesteps, sat_platform=sat_platform
             )
-
         if not self.train:
             if metadata:
                 return x_imgs, x_meta
@@ -927,6 +925,27 @@ def process_data(self):
         raise NotImplementedError
 
 
+"""
+
+read data
+(several option)
+but return alwasys this shape
+
+
+process data
+a lot of options but shuold always be able to do:
+
+upsampling (optional)
+augmentaiton (Optional)
+padding (Optional)
+nan_sorter (Optional)
+cloud sorter (Optional)
+
+return data
+
+"""
+
+
 class Data(Protocol):
     x_tensor: np.array
     y_tensor: np.array
@@ -971,8 +990,31 @@ def padd(self, padding: int = 0, padding_mode="same"):
                 mode=padding_mode,
             )
 
-    def cloud_sort(self):
-        ...
+    def nan_value_sorter(self):
+        # TODO: Working only when timesteps is the first dimension. Solve to general case
+        # to integrate batch.
+        # That is why the loop on batch.
+        ordered_tensor = []
+        for batch_images in self.x_tensor:
+            # Choose and order timesteps by level of nan_value density.
+            # Expects X -> (Timesteps, num_bands, width, height)
+            x_imgs = filters.order_tensor_on_masks(
+                batch_images, self.x_nan_value, max_images=self.x_tensor.timesteps
+            )
+            ordered_tensor.append(x_imgs)
+        self.x_tensor = ordered_tensor
+
+    def cloud_sorter(self):
+        # TODO: Working only after data is read, so it is expecting shape:
+        # NOT working probably.
+        if (
+            SENTINEL_2 in self.platforms or LANDSAT_8 in self.platforms
+        ) and self.cloud_sort:
+            # Now we only use one platform.
+            sat_platform = [f for f in self.platforms][0]
+            self.x_tensor = filters.order_tensor_on_cloud_mask(
+                self.x_tensor, max_images=self.timesteps, sat_platform=sat_platform
+            )
 
     def normalize(self, normalization: float = None):
         if normalization is not None:
@@ -1011,6 +1053,9 @@ def __init__(self, x_shape: XShape1D, y_shape: YShape1D):
         self.x_shape = x_shape
         self.y_shape = y_shape
 
+    def padding(self):
+        pass
+
 
 class LearningMode:
     def __init__():

pixels/generator/types.py

@@ -2,6 +2,7 @@
 from typing import TypeVar
 
 XShape3D = namedtuple("XShape3D", ["batch", "timesteps", "height", "width", "bands"])
+# batch = batch * timesteps
 XShape2D = namedtuple("XShape2D", ["batch", "height", "width", "bands"])
 XShape1D = namedtuple("XShape1D", ["batch", "timesteps", "bands"])
 
@@ -10,3 +11,5 @@
 
 XShape = TypeVar("XShape", XShape1D, XShape2D, XShape3D)
 YShape = TypeVar("YShape", YShape1D, YShapeND)
+
+# RESNET -> Batch, batch * (number of moving windoe on each batch)