diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 0000000..7e88cbd --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,21 @@ +name: ruff_push +on: [push] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - run: | + python -m pip install --upgrade pip + pip install ruff + - run: ruff check --output-format=github . + - name: If needed, commit ruff changes to a new pull request + if: failure() + run: | + ruff check --output-format=github --fix . + git config --global user.name github-actions + git config --global user.email '${GITHUB_ACTOR}@users.noreply.github.com' + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY + git commit -am "fixup! Format Python code with ruff push" + git push --force origin HEAD:$GITHUB_REF diff --git a/.gitignore b/.gitignore index c6a8d4d..329661f 100644 --- a/.gitignore +++ b/.gitignore @@ -136,4 +136,12 @@ zappa_settings.toml _version.py Makefile -tests/ \ No newline at end of file +tests/ + +test_data/fusion_results +test_data/S2/processed +test_data/S3/reprojected +test_data/S3/composites +test_data/S3/calibrated +test_data/S3/binning +test_data/S3/blurred diff --git a/README.md b/README.md index 54dda51..cf8f9cf 100644 --- a/README.md +++ b/README.md @@ -40,22 +40,36 @@ as demonstrated by the example of Aarhus, Denmark in Spring 2021. See run_efast.py for an example using data located in test_data folder. ### Requirements +* [python](https://www.python.org/getit/) +* [esa-snap](https://step.esa.int/main/download/snap-download/) - needed for Sentinel-3 pre-processing only. Tested with version 9 and 10. -- setuptools -- numpy -- scipy -- tqdm -- scikit-learn -- rasterio -- pandas -- ipdb -- astropy -- python-dateutil -- snap-graph (available through a Git repository) +### Try it out + +1. Clone the repository to your local machine. +2. Navigate to the root directory of the repository in your terminal. +3. [OPTIONAL but recommended] Create a virtual environment: `python3. -m venv .venv` +3. Install the package: `pip install -e .` +4. Run the example: `python run_efast.py` ### Installation +Install the package using pip: + +```bash +pip install git+https://github.com/DHI-GRAS/efast.git +``` + +### Usage +```python +import pyefast + +... +pyefast.fusion( + ... +) +``` +### Develop 1. Clone the repository to your local machine. 2. Navigate to the root directory of the repository in your terminal. -3. Run the following command to install the required packages: pip install -r requirements.txt -4. Run the following command to install the package: python setup.py install +3. [OPTIONAL but strongly recommended] Create a virtual environment: `python3. -m venv .venv` +3. Install the package in dev mode: `pip install -e .[dev]` diff --git a/pyefast/__init__.py b/pyefast/__init__.py index e69de29..1e23566 100644 --- a/pyefast/__init__.py +++ b/pyefast/__init__.py @@ -0,0 +1 @@ +from .efast import fusion diff --git a/pyefast/efast.py b/pyefast/efast.py index c526d59..a25efc2 100644 --- a/pyefast/efast.py +++ b/pyefast/efast.py @@ -26,15 +26,16 @@ """ import os -from tqdm import tqdm import numpy as np import pandas as pd import rasterio import rasterio.windows -from scipy.interpolate import interp1d import scipy.ndimage +from scipy.interpolate import interp1d +from tqdm import tqdm + def fusion( pred_date, diff --git a/pyefast/s2_processing.py b/pyefast/s2_processing.py index b843b99..c3b8dfb 100644 --- a/pyefast/s2_processing.py +++ b/pyefast/s2_processing.py @@ -26,16 +26,16 @@ """ import re -from tqdm import tqdm import xml.etree.ElementTree as ET import numpy as np import pyproj import rasterio import scipy as sp + from shapely.geometry import box from shapely.ops import transform - +from tqdm import tqdm # Mapping of Sentinel-2 bands names to bands ids BANDS_IDS = { @@ -52,10 +52,9 @@ } -def extract_mask_s2_bands(input_dir, - output_dir, - bands=["B02", "B03", "B04", "B8A"], - resolution=20): +def extract_mask_s2_bands( + input_dir, output_dir, bands=["B02", "B03", "B04", "B8A"], resolution=20 +): """ Extract specified Sentinel-2 bands from .SAFE file, mask clouds and shadows using the SLC mask and save to multi-band GeoTIFF file. @@ -79,7 +78,8 @@ def extract_mask_s2_bands(input_dir, """ for p in input_dir.iterdir(): band_paths = [ - list(p.glob(f"GRANULE/*/IMG_DATA/R{resolution}m/*{band}*.jp2"))[0] for band in bands + list(p.glob(f"GRANULE/*/IMG_DATA/R{resolution}m/*{band}*.jp2"))[0] + for band in bands ] # Find S2 BOA offsets @@ -99,7 +99,9 @@ def extract_mask_s2_bands(input_dir, mask = (mask == 0) | (mask == 3) | (mask > 7) # Combine bands and mask - s2_image = np.zeros((len(bands), profile["height"], profile["width"]), "float32") + s2_image = np.zeros( + (len(bands), profile["height"], profile["width"]), "float32" + ) for i, band_path in enumerate(band_paths): band = bands[i] band_id = BANDS_IDS.get(band) @@ -112,7 +114,9 @@ def extract_mask_s2_bands(input_dir, s2_image[i] = data # Save file - profile.update({"driver": "GTiff", "count": len(bands), "dtype": "float32", "nodata": 0}) + profile.update( + {"driver": "GTiff", "count": len(bands), "dtype": "float32", "nodata": 0} + ) out_path = output_dir / f"{str(p.name).rstrip('.SAFE')}_REFL.tif" with rasterio.open(out_path, "w", **profile) as dst: dst.write(s2_image) @@ -170,7 +174,9 @@ def distance_to_clouds(dir_s2, ratio=30, tolerance_percentage=0.05): distance_to_cloud = np.clip(distance_to_cloud, 0, 255) # Update transform - s2_resolution = (s2_profile["transform"]*(1, 0))[0] - (s2_profile["transform"]*(0, 0))[0] + s2_resolution = (s2_profile["transform"] * (1, 0))[0] - ( + s2_profile["transform"] * (0, 0) + )[0] longitude_origin, latitude_origin = s2_profile["transform"] * (0, 0) lr_transform = rasterio.Affine( ratio * s2_resolution, @@ -226,9 +232,9 @@ def get_wkt_footprint(dir_s2, crs="EPSG:4326"): # Ensure footprint is in desired CRS polygon = box(*bounds) if image_crs != crs: - transformer = pyproj.Transformer.from_proj(pyproj.Proj(image_crs), - pyproj.Proj(crs), - always_xy=True) + transformer = pyproj.Transformer.from_proj( + pyproj.Proj(image_crs), pyproj.Proj(crs), always_xy=True + ) polygon = transform(transformer.transform, polygon) # Step 4: Convert to WKT diff --git a/pyefast/s3_processing.py b/pyefast/s3_processing.py index 5ee8960..3f72546 100644 --- a/pyefast/s3_processing.py +++ b/pyefast/s3_processing.py @@ -25,22 +25,23 @@ @author: rmgu, pase """ -from dateutil import rrule -from datetime import datetime import os import re -from tqdm import tqdm + +from datetime import datetime import astropy.convolution as ap import numpy as np import pandas as pd import rasterio -from rasterio.vrt import WarpedVRT -from rasterio.enums import Resampling -from rasterio import shutil as rio_shutil import scipy as sp +from dateutil import rrule +from rasterio import shutil as rio_shutil +from rasterio.enums import Resampling +from rasterio.vrt import WarpedVRT from snap_graph.snap_graph import SnapGraph +from tqdm import tqdm def binning_s3( @@ -100,10 +101,9 @@ def binning_s3( sen3_paths = [element for _, element in sorted(zip(date_strings, sen3_paths))] for i, sen3_path in enumerate(sen3_paths): - output_path = os.path.join( binning_dir, - sen3_path.stem+'.tif', + sen3_path.stem + ".tif", ) variables = s3_bands.copy() @@ -185,13 +185,7 @@ def binning_s3( def produce_median_composite( - dir_s3, - composite_dir, - step=5, - mosaic_days=100, - s3_bands=None, - D=20, - sigma_doy=10 + dir_s3, composite_dir, step=5, mosaic_days=100, s3_bands=None, D=20, sigma_doy=10 ): """ Create weighted composites of Sentinel-3 images. @@ -220,7 +214,10 @@ def produce_median_composite( """ sen3_paths = list(dir_s3.glob("S3*.tif")) s3_dates = pd.to_datetime( - [re.match(".*__(\d{8})T.*\.tif", sen3_path.name).group(1) for sen3_path in sen3_paths] + [ + re.match(".*__(\d{8})T.*\.tif", sen3_path.name).group(1) + for sen3_path in sen3_paths + ] ) sen3_paths = np.array( [sen3_path for _, sen3_path in sorted(zip(s3_dates, sen3_paths))] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..028da26 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pyefast" +authors = [ + {name = "sa", email = "smth@email.com"}, +] +description = "My package description" +readme = "README.md" +dynamic = ["version"] +dependencies = [ + "python-dateutil", + "numpy", + "pandas", + "rasterio", + "scipy", + "tqdm", + "pyproj", + "shapely", + "astropy", + "snap-graph @ git+https://github.com/DHI-GRAS/snap-graph", + +] + +[project.optional-dependencies] +dev = [ + "ruff", +] + +[tool.setuptools.packages.find] +include = ["pyefast"] + +[tool.ruff.lint] +select = ["I"] + +[tool.ruff.lint.isort] +# Use a single line between direct and from import. +lines-between-types = 1 diff --git a/run_efast.py b/run_efast.py index 6a418f9..41c663f 100644 --- a/run_efast.py +++ b/run_efast.py @@ -25,16 +25,16 @@ @author: rmgu, pase """ +import argparse + from datetime import datetime, timedelta from pathlib import Path -import argparse + from dateutil import rrule -# Import s3_fusion modules -import pyefast.s3_processing as s3 -import pyefast.s2_processing as s2 import pyefast.efast as efast - +import pyefast.s2_processing as s2 +import pyefast.s3_processing as s3 # Test parameters path = Path("./test_data").absolute() @@ -50,15 +50,15 @@ def main( - start_date: str, - end_date: str, - s3_sensor: str, - s3_bands: list, - s2_bands: list, - mosaic_days: int, - step: int, + start_date: str, + end_date: str, + s3_sensor: str, + s3_bands: list, + s2_bands: list, + mosaic_days: int, + step: int, + snap_gpt_path: str = "gpt", ): - # Transform parameters start_date = datetime.strptime(start_date, "%Y-%m-%d") end_date = datetime.strptime(end_date, "%Y-%m-%d") @@ -82,23 +82,25 @@ def main( # Sentinel-2 pre-processing s2.extract_mask_s2_bands( s2_download_dir, - s2_processed_dir + s2_processed_dir, + bands=s2_bands, ) s2.distance_to_clouds( - s2_processed_dir + s2_processed_dir, ) footprint = s2.get_wkt_footprint( - s2_processed_dir + s2_processed_dir, ) # Sentinel-3 pre-processing s3.binning_s3( s3_download_dir, s3_binning_dir, - aggregator="mean", + footprint=footprint, s3_bands=s3_bands, instrument=instrument, - footprint=footprint, + aggregator="mean", + snap_gpt_path=snap_gpt_path, snap_memory="24G", snap_parallelization=1, ) @@ -113,7 +115,7 @@ def main( s3_composites_dir, s3_blured_dir, std=1, - preserve_nan=False + preserve_nan=False, ) s3.reformat_s3( s3_blured_dir, @@ -122,7 +124,7 @@ def main( s3.reproject_and_crop_s3( s3_calibrated_dir, s2_processed_dir, - s3_reprojected_dir + s3_reprojected_dir, ) # Perform EFAST fusion @@ -139,7 +141,7 @@ def main( fusion_dir, product="REFL", max_days=100, - minimum_acquisition_importance=0 + minimum_acquisition_importance=0, ) @@ -148,10 +150,13 @@ def main( parser.add_argument("--start-date", default="2023-09-11") parser.add_argument("--end-date", default="2023-09-21") parser.add_argument("--s3-sensor", default="SYN") - parser.add_argument("--s3-bands", default=["SDR_Oa04", "SDR_Oa06", "SDR_Oa08", "SDR_Oa17"]) + parser.add_argument( + "--s3-bands", default=["SDR_Oa04", "SDR_Oa06", "SDR_Oa08", "SDR_Oa17"] + ) parser.add_argument("--s2-bands", default=["B02", "B03", "B04", "B8A"]) parser.add_argument("--mosaic-days", default=100) parser.add_argument("--step", required=False, default=2) + parser.add_argument("--snap-gpt-path", required=False, default="gpt") args = parser.parse_args() @@ -163,4 +168,5 @@ def main( s2_bands=args.s2_bands, step=args.step, mosaic_days=args.mosaic_days, + snap_gpt_path=args.snap_gpt_path, ) diff --git a/setup.py b/setup.py deleted file mode 100644 index 836e57f..0000000 --- a/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Tue Jun 28 15:22:39 2022 - -@author: rmgu -""" - -from setuptools import setup, find_packages - -setup(name="efast", packages=find_packages())