From 2f933963badcc5d18ce79f13ba9d011e37ade427 Mon Sep 17 00:00:00 2001 From: charalamm Date: Fri, 7 Jun 2024 14:13:10 +0200 Subject: [PATCH 1/6] Use the new pytporject.toml instead of the deprecated setup.py Format the respository according to PEP8 Gitignore the porcess results Imporve and update instructions --- .gitignore | 10 ++++++- README.md | 38 ++++++++++++++---------- pyefast/__init__.py | 1 + pyefast/efast.py | 4 +-- pyefast/s2_processing.py | 31 +++++++++++-------- pyefast/s3_processing.py | 29 ++++++++---------- pyproject.toml | 38 ++++++++++++++++++++++++ run_efast.py | 64 +++++++++++++++++----------------------- setup.py | 10 ------- 9 files changed, 130 insertions(+), 95 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index c6a8d4d..329661f 100644 --- a/.gitignore +++ b/.gitignore @@ -136,4 +136,12 @@ zappa_settings.toml _version.py Makefile -tests/ \ No newline at end of file +tests/ + +test_data/fusion_results +test_data/S2/processed +test_data/S3/reprojected +test_data/S3/composites +test_data/S3/calibrated +test_data/S3/binning +test_data/S3/blurred diff --git a/README.md b/README.md index 54dda51..21c7c6c 100644 --- a/README.md +++ b/README.md @@ -40,22 +40,30 @@ as demonstrated by the example of Aarhus, Denmark in Spring 2021. See run_efast.py for an example using data located in test_data folder. ### Requirements +* [python](https://www.python.org/getit/) +* [esa-snap](https://step.esa.int/main/download/snap-download/) -- setuptools -- numpy -- scipy -- tqdm -- scikit-learn -- rasterio -- pandas -- ipdb -- astropy -- python-dateutil -- snap-graph (available through a Git repository) - -### Installation +### Try it out 1. Clone the repository to your local machine. 2. Navigate to the root directory of the repository in your terminal. -3. Run the following command to install the required packages: pip install -r requirements.txt -4. Run the following command to install the package: python setup.py install +3. [OPTIONAL but recommended] Create a virtual environment: `python3. -m venv venv` +3. Install the package: `pip install -e ./` +4. Run the example: `python run_efast.py` + +### Installation +Install the package using pip: + +```bash +pip install git+https://github.com/DHI-GRAS/efast.git +``` + +### Usage +```python +import pyefast + +... +pyefast.fusion( + ... +) +``` diff --git a/pyefast/__init__.py b/pyefast/__init__.py index e69de29..1e23566 100644 --- a/pyefast/__init__.py +++ b/pyefast/__init__.py @@ -0,0 +1 @@ +from .efast import fusion diff --git a/pyefast/efast.py b/pyefast/efast.py index c526d59..2325c9b 100644 --- a/pyefast/efast.py +++ b/pyefast/efast.py @@ -26,14 +26,14 @@ """ import os -from tqdm import tqdm import numpy as np import pandas as pd import rasterio import rasterio.windows -from scipy.interpolate import interp1d import scipy.ndimage +from scipy.interpolate import interp1d +from tqdm import tqdm def fusion( diff --git a/pyefast/s2_processing.py b/pyefast/s2_processing.py index b843b99..f82bba5 100644 --- a/pyefast/s2_processing.py +++ b/pyefast/s2_processing.py @@ -26,7 +26,6 @@ """ import re -from tqdm import tqdm import xml.etree.ElementTree as ET import numpy as np @@ -35,7 +34,7 @@ import scipy as sp from shapely.geometry import box from shapely.ops import transform - +from tqdm import tqdm # Mapping of Sentinel-2 bands names to bands ids BANDS_IDS = { @@ -52,10 +51,9 @@ } -def extract_mask_s2_bands(input_dir, - output_dir, - bands=["B02", "B03", "B04", "B8A"], - resolution=20): +def extract_mask_s2_bands( + input_dir, output_dir, bands=["B02", "B03", "B04", "B8A"], resolution=20 +): """ Extract specified Sentinel-2 bands from .SAFE file, mask clouds and shadows using the SLC mask and save to multi-band GeoTIFF file. @@ -79,7 +77,8 @@ def extract_mask_s2_bands(input_dir, """ for p in input_dir.iterdir(): band_paths = [ - list(p.glob(f"GRANULE/*/IMG_DATA/R{resolution}m/*{band}*.jp2"))[0] for band in bands + list(p.glob(f"GRANULE/*/IMG_DATA/R{resolution}m/*{band}*.jp2"))[0] + for band in bands ] # Find S2 BOA offsets @@ -99,7 +98,9 @@ def extract_mask_s2_bands(input_dir, mask = (mask == 0) | (mask == 3) | (mask > 7) # Combine bands and mask - s2_image = np.zeros((len(bands), profile["height"], profile["width"]), "float32") + s2_image = np.zeros( + (len(bands), profile["height"], profile["width"]), "float32" + ) for i, band_path in enumerate(band_paths): band = bands[i] band_id = BANDS_IDS.get(band) @@ -112,7 +113,9 @@ def extract_mask_s2_bands(input_dir, s2_image[i] = data # Save file - profile.update({"driver": "GTiff", "count": len(bands), "dtype": "float32", "nodata": 0}) + profile.update( + {"driver": "GTiff", "count": len(bands), "dtype": "float32", "nodata": 0} + ) out_path = output_dir / f"{str(p.name).rstrip('.SAFE')}_REFL.tif" with rasterio.open(out_path, "w", **profile) as dst: dst.write(s2_image) @@ -170,7 +173,9 @@ def distance_to_clouds(dir_s2, ratio=30, tolerance_percentage=0.05): distance_to_cloud = np.clip(distance_to_cloud, 0, 255) # Update transform - s2_resolution = (s2_profile["transform"]*(1, 0))[0] - (s2_profile["transform"]*(0, 0))[0] + s2_resolution = (s2_profile["transform"] * (1, 0))[0] - ( + s2_profile["transform"] * (0, 0) + )[0] longitude_origin, latitude_origin = s2_profile["transform"] * (0, 0) lr_transform = rasterio.Affine( ratio * s2_resolution, @@ -226,9 +231,9 @@ def get_wkt_footprint(dir_s2, crs="EPSG:4326"): # Ensure footprint is in desired CRS polygon = box(*bounds) if image_crs != crs: - transformer = pyproj.Transformer.from_proj(pyproj.Proj(image_crs), - pyproj.Proj(crs), - always_xy=True) + transformer = pyproj.Transformer.from_proj( + pyproj.Proj(image_crs), pyproj.Proj(crs), always_xy=True + ) polygon = transform(transformer.transform, polygon) # Step 4: Convert to WKT diff --git a/pyefast/s3_processing.py b/pyefast/s3_processing.py index 5ee8960..1027618 100644 --- a/pyefast/s3_processing.py +++ b/pyefast/s3_processing.py @@ -25,22 +25,21 @@ @author: rmgu, pase """ -from dateutil import rrule -from datetime import datetime import os import re -from tqdm import tqdm +from datetime import datetime import astropy.convolution as ap import numpy as np import pandas as pd import rasterio -from rasterio.vrt import WarpedVRT -from rasterio.enums import Resampling -from rasterio import shutil as rio_shutil import scipy as sp - +from dateutil import rrule +from rasterio import shutil as rio_shutil +from rasterio.enums import Resampling +from rasterio.vrt import WarpedVRT from snap_graph.snap_graph import SnapGraph +from tqdm import tqdm def binning_s3( @@ -100,10 +99,9 @@ def binning_s3( sen3_paths = [element for _, element in sorted(zip(date_strings, sen3_paths))] for i, sen3_path in enumerate(sen3_paths): - output_path = os.path.join( binning_dir, - sen3_path.stem+'.tif', + sen3_path.stem + ".tif", ) variables = s3_bands.copy() @@ -185,13 +183,7 @@ def binning_s3( def produce_median_composite( - dir_s3, - composite_dir, - step=5, - mosaic_days=100, - s3_bands=None, - D=20, - sigma_doy=10 + dir_s3, composite_dir, step=5, mosaic_days=100, s3_bands=None, D=20, sigma_doy=10 ): """ Create weighted composites of Sentinel-3 images. @@ -220,7 +212,10 @@ def produce_median_composite( """ sen3_paths = list(dir_s3.glob("S3*.tif")) s3_dates = pd.to_datetime( - [re.match(".*__(\d{8})T.*\.tif", sen3_path.name).group(1) for sen3_path in sen3_paths] + [ + re.match(".*__(\d{8})T.*\.tif", sen3_path.name).group(1) + for sen3_path in sen3_paths + ] ) sen3_paths = np.array( [sen3_path for _, sen3_path in sorted(zip(s3_dates, sen3_paths))] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2e7bdff --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pyefast" +authors = [ + {name = "sa", email = "smth@email.com"}, +] +description = "My package description" +readme = "README.md" +dynamic = ["version"] +dependencies = [ + "python-dateutil", + "numpy", + "pandas", + "rasterio", + "scipy", + "tqdm", + "pyproj", + "shapely", + "astropy", + "snap-graph @ git+https://github.com/DHI-GRAS/snap-graph", + +] + +[project.optional-dependencies] +dev = ["ruff"] + +[tool.setuptools.packages.find] +include = ["pyefast"] + +[tool.ruff] +line-length = 88 +indent-width = 4 + +[tool.ruff.lint] +select = ["I"] diff --git a/run_efast.py b/run_efast.py index 6a418f9..11fd6e3 100644 --- a/run_efast.py +++ b/run_efast.py @@ -25,16 +25,17 @@ @author: rmgu, pase """ +import argparse from datetime import datetime, timedelta from pathlib import Path -import argparse + from dateutil import rrule -# Import s3_fusion modules -import pyefast.s3_processing as s3 -import pyefast.s2_processing as s2 import pyefast.efast as efast +import pyefast.s2_processing as s2 +# Import s3_fusion modules +import pyefast.s3_processing as s3 # Test parameters path = Path("./test_data").absolute() @@ -50,15 +51,15 @@ def main( - start_date: str, - end_date: str, - s3_sensor: str, - s3_bands: list, - s2_bands: list, - mosaic_days: int, - step: int, + start_date: str, + end_date: str, + s3_sensor: str, + s3_bands: list, + s2_bands: list, + mosaic_days: int, + step: int, + snap_gpt_path: str = "gpt", ): - # Transform parameters start_date = datetime.strptime(start_date, "%Y-%m-%d") end_date = datetime.strptime(end_date, "%Y-%m-%d") @@ -80,25 +81,19 @@ def main( folder.mkdir(parents=True, exist_ok=True) # Sentinel-2 pre-processing - s2.extract_mask_s2_bands( - s2_download_dir, - s2_processed_dir - ) - s2.distance_to_clouds( - s2_processed_dir - ) - footprint = s2.get_wkt_footprint( - s2_processed_dir - ) + s2.extract_mask_s2_bands(s2_download_dir, s2_processed_dir, bands=s2_bands) + s2.distance_to_clouds(s2_processed_dir) + footprint = s2.get_wkt_footprint(s2_processed_dir) # Sentinel-3 pre-processing s3.binning_s3( s3_download_dir, s3_binning_dir, - aggregator="mean", + footprint=footprint, s3_bands=s3_bands, instrument=instrument, - footprint=footprint, + aggregator="mean", + snap_gpt_path=snap_gpt_path, snap_memory="24G", snap_parallelization=1, ) @@ -109,21 +104,12 @@ def main( step=step, s3_bands=None, ) - s3.smoothing( - s3_composites_dir, - s3_blured_dir, - std=1, - preserve_nan=False - ) + s3.smoothing(s3_composites_dir, s3_blured_dir, std=1, preserve_nan=False) s3.reformat_s3( s3_blured_dir, s3_calibrated_dir, ) - s3.reproject_and_crop_s3( - s3_calibrated_dir, - s2_processed_dir, - s3_reprojected_dir - ) + s3.reproject_and_crop_s3(s3_calibrated_dir, s2_processed_dir, s3_reprojected_dir) # Perform EFAST fusion for date in rrule.rrule( @@ -139,7 +125,7 @@ def main( fusion_dir, product="REFL", max_days=100, - minimum_acquisition_importance=0 + minimum_acquisition_importance=0, ) @@ -148,10 +134,13 @@ def main( parser.add_argument("--start-date", default="2023-09-11") parser.add_argument("--end-date", default="2023-09-21") parser.add_argument("--s3-sensor", default="SYN") - parser.add_argument("--s3-bands", default=["SDR_Oa04", "SDR_Oa06", "SDR_Oa08", "SDR_Oa17"]) + parser.add_argument( + "--s3-bands", default=["SDR_Oa04", "SDR_Oa06", "SDR_Oa08", "SDR_Oa17"] + ) parser.add_argument("--s2-bands", default=["B02", "B03", "B04", "B8A"]) parser.add_argument("--mosaic-days", default=100) parser.add_argument("--step", required=False, default=2) + parser.add_argument("--snap-gpt-path", required=False, default="gpt") args = parser.parse_args() @@ -163,4 +152,5 @@ def main( s2_bands=args.s2_bands, step=args.step, mosaic_days=args.mosaic_days, + snap_gpt_path=args.snap_gpt_path ) diff --git a/setup.py b/setup.py deleted file mode 100644 index 836e57f..0000000 --- a/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Tue Jun 28 15:22:39 2022 - -@author: rmgu -""" - -from setuptools import setup, find_packages - -setup(name="efast", packages=find_packages()) From 6236d364af46611ae01fc11ebc5c906a7b29e2ee Mon Sep 17 00:00:00 2001 From: Radoslaw Guzinski Date: Mon, 10 Jun 2024 10:29:51 +0200 Subject: [PATCH 2/6] Small cosmetic changes --- README.md | 2 +- run_efast.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 21c7c6c..6d9211a 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ See run_efast.py for an example using data located in test_data folder. ### Requirements * [python](https://www.python.org/getit/) -* [esa-snap](https://step.esa.int/main/download/snap-download/) +* [esa-snap](https://step.esa.int/main/download/snap-download/) - tested with version 9 and 10 ### Try it out diff --git a/run_efast.py b/run_efast.py index 11fd6e3..470f4b8 100644 --- a/run_efast.py +++ b/run_efast.py @@ -33,8 +33,6 @@ import pyefast.efast as efast import pyefast.s2_processing as s2 - -# Import s3_fusion modules import pyefast.s3_processing as s3 # Test parameters From b172922d0647a2c13ebd96abf7bf7c084e083f9a Mon Sep 17 00:00:00 2001 From: Radoslaw Guzinski Date: Mon, 10 Jun 2024 10:32:58 +0200 Subject: [PATCH 3/6] Small readme update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d9211a..a3a993d 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ See run_efast.py for an example using data located in test_data folder. ### Requirements * [python](https://www.python.org/getit/) -* [esa-snap](https://step.esa.int/main/download/snap-download/) - tested with version 9 and 10 +* [esa-snap](https://step.esa.int/main/download/snap-download/) - needed for Sentinel-3 pre-processing only. Tested with version 9 and 10. ### Try it out From 976651782a8382d18de3530a365a3f31577ed5b4 Mon Sep 17 00:00:00 2001 From: charalamm Date: Mon, 10 Jun 2024 12:38:25 +0200 Subject: [PATCH 4/6] Change ruff and add github action --- .github/workflows/ruff.yml | 21 +++++++++++++++++++++ README.md | 10 ++++++++-- pyefast/efast.py | 1 + pyefast/s3_processing.py | 2 ++ pyproject.toml | 12 +++++++----- run_efast.py | 3 ++- 6 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/ruff.yml diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 0000000..7e88cbd --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,21 @@ +name: ruff_push +on: [push] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - run: | + python -m pip install --upgrade pip + pip install ruff + - run: ruff check --output-format=github . + - name: If needed, commit ruff changes to a new pull request + if: failure() + run: | + ruff check --output-format=github --fix . + git config --global user.name github-actions + git config --global user.email '${GITHUB_ACTOR}@users.noreply.github.com' + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY + git commit -am "fixup! Format Python code with ruff push" + git push --force origin HEAD:$GITHUB_REF diff --git a/README.md b/README.md index a3a993d..cf8f9cf 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,8 @@ See run_efast.py for an example using data located in test_data folder. 1. Clone the repository to your local machine. 2. Navigate to the root directory of the repository in your terminal. -3. [OPTIONAL but recommended] Create a virtual environment: `python3. -m venv venv` -3. Install the package: `pip install -e ./` +3. [OPTIONAL but recommended] Create a virtual environment: `python3. -m venv .venv` +3. Install the package: `pip install -e .` 4. Run the example: `python run_efast.py` ### Installation @@ -67,3 +67,9 @@ pyefast.fusion( ... ) ``` + +### Develop +1. Clone the repository to your local machine. +2. Navigate to the root directory of the repository in your terminal. +3. [OPTIONAL but strongly recommended] Create a virtual environment: `python3. -m venv .venv` +3. Install the package in dev mode: `pip install -e .[dev]` diff --git a/pyefast/efast.py b/pyefast/efast.py index 2325c9b..a25efc2 100644 --- a/pyefast/efast.py +++ b/pyefast/efast.py @@ -32,6 +32,7 @@ import rasterio import rasterio.windows import scipy.ndimage + from scipy.interpolate import interp1d from tqdm import tqdm diff --git a/pyefast/s3_processing.py b/pyefast/s3_processing.py index 1027618..3f72546 100644 --- a/pyefast/s3_processing.py +++ b/pyefast/s3_processing.py @@ -27,6 +27,7 @@ import os import re + from datetime import datetime import astropy.convolution as ap @@ -34,6 +35,7 @@ import pandas as pd import rasterio import scipy as sp + from dateutil import rrule from rasterio import shutil as rio_shutil from rasterio.enums import Resampling diff --git a/pyproject.toml b/pyproject.toml index 2e7bdff..028da26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,14 +25,16 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["ruff"] +dev = [ + "ruff", +] [tool.setuptools.packages.find] include = ["pyefast"] -[tool.ruff] -line-length = 88 -indent-width = 4 - [tool.ruff.lint] select = ["I"] + +[tool.ruff.lint.isort] +# Use a single line between direct and from import. +lines-between-types = 1 diff --git a/run_efast.py b/run_efast.py index 470f4b8..cb1b864 100644 --- a/run_efast.py +++ b/run_efast.py @@ -26,6 +26,7 @@ """ import argparse + from datetime import datetime, timedelta from pathlib import Path @@ -150,5 +151,5 @@ def main( s2_bands=args.s2_bands, step=args.step, mosaic_days=args.mosaic_days, - snap_gpt_path=args.snap_gpt_path + snap_gpt_path=args.snap_gpt_path, ) From 53e9795785014a3c7f77ebc61f906c69e565fafd Mon Sep 17 00:00:00 2001 From: github-actions <${GITHUB_ACTOR}@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:01:59 +0000 Subject: [PATCH 5/6] fixup! Format Python code with ruff push --- pyefast/s2_processing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyefast/s2_processing.py b/pyefast/s2_processing.py index f82bba5..c3b8dfb 100644 --- a/pyefast/s2_processing.py +++ b/pyefast/s2_processing.py @@ -32,6 +32,7 @@ import pyproj import rasterio import scipy as sp + from shapely.geometry import box from shapely.ops import transform from tqdm import tqdm From 531dc3285bc5d0a21aa1def000bcce31b77dce83 Mon Sep 17 00:00:00 2001 From: charalamm Date: Mon, 10 Jun 2024 13:07:57 +0200 Subject: [PATCH 6/6] Clear --- run_efast.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/run_efast.py b/run_efast.py index cb1b864..41c663f 100644 --- a/run_efast.py +++ b/run_efast.py @@ -80,9 +80,17 @@ def main( folder.mkdir(parents=True, exist_ok=True) # Sentinel-2 pre-processing - s2.extract_mask_s2_bands(s2_download_dir, s2_processed_dir, bands=s2_bands) - s2.distance_to_clouds(s2_processed_dir) - footprint = s2.get_wkt_footprint(s2_processed_dir) + s2.extract_mask_s2_bands( + s2_download_dir, + s2_processed_dir, + bands=s2_bands, + ) + s2.distance_to_clouds( + s2_processed_dir, + ) + footprint = s2.get_wkt_footprint( + s2_processed_dir, + ) # Sentinel-3 pre-processing s3.binning_s3( @@ -103,12 +111,21 @@ def main( step=step, s3_bands=None, ) - s3.smoothing(s3_composites_dir, s3_blured_dir, std=1, preserve_nan=False) + s3.smoothing( + s3_composites_dir, + s3_blured_dir, + std=1, + preserve_nan=False, + ) s3.reformat_s3( s3_blured_dir, s3_calibrated_dir, ) - s3.reproject_and_crop_s3(s3_calibrated_dir, s2_processed_dir, s3_reprojected_dir) + s3.reproject_and_crop_s3( + s3_calibrated_dir, + s2_processed_dir, + s3_reprojected_dir, + ) # Perform EFAST fusion for date in rrule.rrule(