Merge branch 'master' into issue-287

Author: vss-2

.github/workflows/Python-CMD-check.yaml

@@ -0,0 +1,67 @@
+on:
+  push:
+    paths:
+      - python-package/**
+    branches:
+      - main
+      - master
+      - dev
+  pull_request:
+    paths:
+      - python-package/**
+    branches:
+      - main
+      - master
+      - dev
+
+name: Python-CMD-check
+
+jobs:
+  Python-CMD-check:
+    runs-on: ${{ matrix.os }}
+
+    name: ${{ matrix.os }} (${{ matrix.python-version }})
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+        python-version: ["3.9", "3.10", "3.11"]
+        # Python 3.8 support ends in 2024-10
+        # Python 3.12 support starts in 2023-10
+        # Check Python maintenance status at: https://www.python.org/downloads/
+
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - name: Check out geobr
+        uses: actions/checkout@v3
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install Poetry
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+
+      - name: Install dependencies with Poetry
+        run: poetry install
+        working-directory: python-package
+
+      - name: Run tests with Poetry
+        run: |
+          poetry run pytest ./tests
+        working-directory: python-package
+
+      - name: Upload check results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: python-package/test-results.txt
+          if-no-files-found: warn

.github/workflows/R-CMD-check-CRAN.yaml

@@ -0,0 +1,88 @@
+# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag.
+# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions
+on:
+  push:
+    paths:
+      - r-package/**
+    branches:
+      - main
+      - dev
+  pull_request:
+    paths:
+      - r-package/**
+    branches:
+      - main
+      - dev
+
+name: R-CMD-check-as-CRAN
+
+jobs:
+  R-CMD-check:
+    runs-on: ${{ matrix.config.os }}
+
+    name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
+
+    env:
+      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
+      RSPM: ${{ matrix.config.rspm }}
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: ${{ matrix.config.r }}
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - name: Query dependencies
+        run: |
+          install.packages('remotes')
+          saveRDS(remotes::dev_package_deps(dependencies = TRUE), "../.github/depends.Rds", version = 2)
+          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), "../.github/R-version")
+        shell: Rscript {0}
+        working-directory: r-package
+
+      - name: Cache R packages
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.R_LIBS_USER }}
+          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
+
+      - name: Install system dependencies
+        run: |
+          while read -r cmd
+          do
+            eval sudo $cmd
+          done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
+        working-directory: r-package
+
+      - name: Install dependencies
+        run: |
+          remotes::install_deps(dependencies = TRUE)
+          remotes::install_cran("rcmdcheck")
+        shell: Rscript {0}
+        working-directory: r-package
+
+      - name: Check
+        env:
+          _R_CHECK_CRAN_INCOMING_REMOTE_: false
+          NOT_CRAN: false
+        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check")
+        shell: Rscript {0}
+        working-directory: r-package
+
+      - name: Upload check results
+        if: failure()
+        uses: actions/upload-artifact@main
+        with:
+          name: ${{ runner.os }}-r${{ matrix.config.r }}-results
+          path: check

.github/workflows/R-CMD-check.yaml

@@ -31,7 +31,7 @@ jobs:
         - {os: windows-latest, r: 'release'}
         - {os: windows-latest, r: 'oldrel'}
         - {os: macOS-latest, r: 'release'}
-        - {os: macOS-latest, r: 'oldrel'}
+       # - {os: macOS-latest, r: 'oldrel'}
         - {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
         - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
         - {os: ubuntu-20.04, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
@@ -44,11 +44,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - uses: r-lib/actions/setup-r@v1
+      - uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.r }}
 
-      - uses: r-lib/actions/setup-pandoc@v1
+      - uses: r-lib/actions/setup-pandoc@v2
 
       - name: Query dependencies
         run: |

.github/workflows/pkgdown.yml

@@ -13,9 +13,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - uses: r-lib/actions/setup-r@v1
+      - uses: r-lib/actions/setup-r@v2
 
-      - uses: r-lib/actions/setup-pandoc@v1
+      - uses: r-lib/actions/setup-pandoc@v2
 
       - name: Query dependencies
         run: |

.github/workflows/test-coverage.yaml

@@ -24,7 +24,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - uses: r-lib/actions/setup-r@v1
+      - uses: r-lib/actions/setup-r@v2
         id: install-r
 
       - name: Install pak and query dependencies

.gitignore

@@ -8,14 +8,18 @@
 *.out
 *.toc
 *.ind
+*.gpkg
+*.rds
 *.Rhistory
 /R/.Rhistory
 /r-package/.Rhistory
 /.Rhistory
 /man/figures/.Rhistory
 /r-package/.Rhistory
 /prep_data/.Rhistory
-/data-raw/
+/data_prep/data/*
+/data_prep/data_raw/*
+/data_prep/_targets/*
 
 inst/doc
 Meta

README.md

@@ -11,7 +11,8 @@ The package is currently available in [**R**](https://CRAN.R-project.org/package
 
 | ***R*** | ***Python*** | ***Repo***|
 |-----|-----|----|
-| [![CRAN/METACRAN Version](https://www.r-pkg.org/badges/version/geobr)](https://CRAN.R-project.org/package=geobr) <br /> [![CRAN/METACRAN Total downloads](http://cranlogs.r-pkg.org/badges/grand-total/geobr?color=blue)](https://CRAN.R-project.org/package=geobr) <br /> [![CRAN/METACRAN downloads per month](http://cranlogs.r-pkg.org/badges/geobr?color=yellow)](https://CRAN.R-project.org/package=geobr) <br /> [![Codecov test coverage](https://codecov.io/gh/ipeaGIT/geobr/branch/master/graph/badge.svg)](https://app.codecov.io/gh/ipeaGIT/geobr?branch=master)  <br /> [![Lifecycle: stable](https://img.shields.io/badge/lifecycle-stable-brightgreen.svg)](https://www.tidyverse.org/lifecycle/#stable) | [![PyPI version](https://badge.fury.io/py/geobr.svg)](https://badge.fury.io/py/geobr) <br />  [![Downloads](https://pepy.tech/badge/geobr)](https://pepy.tech/project/geobr) <br />  [![Downloads](https://pepy.tech/badge/geobr/month)](https://pepy.tech/project/geobr/month)  <br /> [![Lifecycle: maturing](https://img.shields.io/badge/lifecycle-maturing-blue.svg)](https://www.tidyverse.org/lifecycle/#maturing) |<img alt="GitHub stars" src="https://img.shields.io/github/stars/ipeaGIT/geobr.svg?color=orange"> <br /> [![R build status](https://github.com/ipeaGIT/geobr/workflows/R-CMD-check/badge.svg)](https://github.com/ipeaGIT/geobr/actions) <br />  [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) |
+| [![CRAN/METACRAN Version](https://www.r-pkg.org/badges/version/geobr)](https://CRAN.R-project.org/package=geobr) <br /> [![CRAN/METACRAN Total downloads](http://cranlogs.r-pkg.org/badges/grand-total/geobr?color=blue)](https://CRAN.R-project.org/package=geobr) <br /> [![CRAN/METACRAN downloads per month](http://cranlogs.r-pkg.org/badges/geobr?color=yellow)](https://CRAN.R-project.org/package=geobr) <br /> [![Codecov test coverage](https://codecov.io/gh/ipeaGIT/geobr/branch/master/graph/badge.svg)](https://app.codecov.io/gh/ipeaGIT/geobr?branch=master)  <br /> [![Lifecycle: stable](https://img.shields.io/badge/lifecycle-stable-brightgreen.svg)](https://www.tidyverse.org/lifecycle/#stable) <br /> [![R build status](https://github.com/ipeaGIT/geobr/workflows/R-CMD-check/badge.svg)](https://github.com/ipeaGIT/geobr/actions)  | [![PyPI version](https://badge.fury.io/py/geobr.svg)](https://badge.fury.io/py/geobr) <br />  [![Downloads](https://static.pepy.tech/badge/geobr)](https://pepy.tech/project/geobr) <br />  [![Downloads](https://static.pepy.tech/badge/geobr/month)](https://pepy.tech/project/geobr)  <br /> [![Lifecycle: maturing](https://img.shields.io/badge/lifecycle-maturing-blue.svg)](https://www.tidyverse.org/lifecycle/#maturing) <br /> [![Python build status](https://github.com/ipeaGIT/geobr/workflows/Python-CMD-check/badge.svg)](https://github.com/ipeaGIT/geobr/actions) |<img alt="GitHub stars" src="https://img.shields.io/github/stars/ipeaGIT/geobr.svg?color=orange"> <br /> <br />  [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) |
+
 
 
 
@@ -95,7 +96,7 @@ More examples [here](https://github.com/ipeaGIT/geobr/tree/master/python-package
 |`read_micro_region`| Micro region | 2000, 2001, 2010, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 | IBGE |
 |`read_intermediate_region`| Intermediate region | 2017, 2019, 2020 |  IBGE |
 |`read_immediate_region`| Immediate region | 2017, 2019, 2020 |  IBGE |
-|`read_municipality`| Municipality | 1872, 1900, 1911, 1920, 1933, 1940, 1950, 1960, 1970, 1980, 1991, 2000, 2001, 2005, 2007, 2010, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 | IBGE |
+|`read_municipality`| Municipality | 1872, 1900, 1911, 1920, 1933, 1940, 1950, 1960, 1970, 1980, 1991, 2000, 2001, 2005, 2007, 2010, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 | IBGE |
 |`read_municipal_seat`| Municipality seats (sedes municipais) |  1872, 1900, 1911, 1920, 1933, 1940, 1950, 1960, 1970, 1980, 1991, 2010 | IBGE |
 |`read_weighting_area`| Census weighting area (área de ponderação) |  2010 | IBGE |
 |`read_census_tract`| Census tract (setor censitário) |  2000, 2010, 2017, 2019, 2020 | IBGE |
@@ -108,10 +109,10 @@ More examples [here](https://github.com/ipeaGIT/geobr/tree/master/python-package
 |`read_disaster_risk_area` | Disaster risk areas | 2010 | CEMADEN and IBGE | 
 |`read_indigenous_land` | Indigenous lands | 201907, 202103 | FUNAI | 
 |`read_semiarid` | Semi Arid region | 2005, 2017 | IBGE | 
-|`read_health_facilities` | Health facilities | 2015 | CNES, DataSUS | 
+|`read_health_facilities` | Health facilities | 201505, 202303 | CNES, DataSUS | 
 |`read_health_region` | Health regions and macro regions | 1991, 1994, 1997, 2001, 2005, 2013 | DataSUS |
 |`read_neighborhood` | Neighborhood limits |  2010 | IBGE |
-|`read_schools` | Schools |  2020 | INEP |
+|`read_schools` | Schools |  2020, 2023 | INEP |
 |`read_comparable_areas` | Historically comparable municipalities, aka Areas minimas comparaveis (AMCs) |  1872,1900,1911,1920,1933,1940,1950,1960,1970,1980,1991,2000,2010 | IBGE |
 |`read_urban_concentrations` | Urban concentration areas (concentrações urbanas) | 2015 | IBGE | 
 |`read_pop_arrangements` | Population arrangements (arranjos populacioanis) | 2015 | IBGE | 

data_prep/.RData

@@ -0,0 +1,123 @@
+library(sf)
+library(data.table)
+library(dplyr)
+
+
+year <- 2022
+
+# create dest dir
+raw_dir <- paste0('./data_raw/census_tracts/',year)
+dest_dir <- paste0('./data/census_tracts/',year)
+dir.create(raw_dir, recursive = T)
+dir.create(dest_dir, recursive = T)
+
+
+
+
+#### 0. Download original data sets from IBGE ftp -----------------
+
+if(year == 2022){
+
+  dest_file <-  download_file(file_url = ftp, dest_dir = raw_dir)
+
+}
+
+
+#### 1. unzip -----------------
+
+  temp_dir <- tempdir()
+
+  unzip(dest_file, exdir = temp_dir)
+
+  local_file <- unzip_fun(dest_file)
+
+
+
+df <- sf::st_read('./data_raw/census_tracts/2022/BR_Malha_Preliminar_2022.gpkg')
+saveRDS(df, './data_raw/census_tracts/2022/BR_Malha_Preliminar_2022.rds')
+
+
+
+#### 1. clean and save data -----------------
+
+df$AREA_KM2 <- NULL
+
+temp_sf <- dplyr::rename(df,
+                     code_tract = CD_SETOR,
+                     code_muni = CD_MUN,
+                     name_muni = NM_MUN,
+                     code_subdistrict = CD_SUBDIST,
+                     name_subdistrict = NM_SUBDIST,
+                     code_district = CD_DIST,
+                     name_district = NM_DIST,
+                     code_urban_concentration = CD_CONCURB,
+                     name_urban_concentration = NM_CONCURB,
+                     code_state = CD_UF,
+                     name_state = NM_UF,
+                     code_micro = CD_MICRO,
+                     name_micro = NM_MICRO,
+                     code_meso = CD_MESO,
+                     name_meso = NM_MESO,
+                     code_immediate = CD_RGI,
+                     name_immediate = NM_RGI,
+                     code_intermediate = CD_RGINT,
+                     name_intermediate = NM_RGINT,
+                     code_region = CD_REGIAO,
+                     name_region = NM_REGIAO
+                     )
+head(temp_sf)
+
+
+# remove P from code tract
+temp_sf <- mutate(temp_sf, code_tract = gsub("P","", code_tract))
+head(temp_sf)
+
+
+# make all columns as character
+char_cols <- names(temp_sf)
+char_cols <- char_cols[char_cols %like% 'code_|name_']
+temp_sf <- mutate(temp_sf, across(all_of(char_cols), as.character))
+
+
+# Use UTF-8 encoding
+temp_sf <- use_encoding_utf8(temp_sf)
+
+# Harmonize spatial projection CRS, using SIRGAS 2000 epsg (SRID): 4674
+temp_sf <- harmonize_projection(temp_sf)
+gc()
+
+
+# harmonize and save
+
+save_state <- function(code_uf){ # code_uf <- 33
+
+    temp_sf2 <- subset(temp_sf, code_state == code_uf)
+    # temp_sf2 <- subset(temp_sf2, code_muni == '3304557')
+
+    # convert to MULTIPOLYGON
+    temp_sf2 <- to_multipolygon(temp_sf2)
+
+    # simplify
+    temp_sf_simplified <- simplify_temp_sf(temp_sf2, tolerance = 10)
+
+    # Save cleaned sf in the cleaned directory
+    sf::st_write(temp_sf2, paste0(dest_dir,'/', code_uf,'census_tract_', year, '.gpkg'))
+    sf::st_write(temp_sf_simplified, paste0(dest_dir,'/', code_uf,'census_tract_', year, '_simplified.gpkg'))
+
+  }
+
+
+all_states <- unique(temp_sf$code_state)
+
+# Apply function to save the data
+gc(reset = T)
+
+# tictoc::tic()
+# future::plan(strategy = 'multisession')
+# furrr::future_map(.x=all_states, .f=save_state, .progress = T)
+# tictoc::toc()
+
+
+tictoc::tic()
+pbapply::pblapply(X=all_states, FUN=save_state)
+tictoc::toc()