refactor data processing #808
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: commit | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| jobs: | |
| build_and_test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-22.04, macos-latest, windows-2019] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: taiki-e/install-action@cargo-hack | |
| - uses: arduino/setup-protoc@v2 | |
| with: | |
| version: "23.1" | |
| repo-token: ${{ secrets.DEV_GITHUB_TOKEN }} | |
| - uses: dtolnay/[email protected] | |
| with: | |
| components: clippy | |
| - uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); | |
| core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') | |
| - name: Env | |
| run: | | |
| env|sort | |
| - name: XCode env | |
| if: runner.os == 'macOS' | |
| run: xcodebuild -version | |
| - name: Build and run tests on windows | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: | | |
| Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser | |
| Invoke-RestMethod -Uri https://get.scoop.sh | Invoke-Expression | |
| scoop install sccache | |
| # Set environment variables | |
| $env:SCCACHE_GHA_ENABLED = "on" | |
| $env:RUSTC_WRAPPER = (Get-Command sccache).Source | |
| $env:XET_GLOBAL_DEDUP_POLICY = "never" | |
| # Change directory and build the Rust project | |
| cd rust | |
| cargo build --verbose --profile=opt-test | |
| # Configure Git global settings | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "XetData Automation" | |
| cargo test --profile=opt-test | |
| # Run intergration tests | |
| cd .. | |
| cd gitxet/ | |
| cargo build --verbose --profile=opt-test | |
| cargo test --profile=opt-test | |
| - name: Lint (Linux only) | |
| if: runner.os == 'Linux' | |
| run: | | |
| rustup component add rustfmt | |
| for d in * ; do | |
| if [[ ! -e "$d/Cargo.toml" ]] ; then | |
| continue | |
| fi | |
| pushd $d | |
| cargo clippy -r --verbose -- -D warnings # elevates warnings to errors | |
| # cargo format. | |
| if ! cargo fmt --check ; then | |
| echo "cargo fmt not run; please run ./scripts/ready-pull-request.sh and resubmit." | |
| exit 1 | |
| fi | |
| # Disabling this for now as a check; fails way too often. | |
| # | |
| # if [[ -e ./Cargo.lock ]] ; then | |
| # # Offline prevents it from updating packages that haven't been updated locally. | |
| # if ! cargo generate-lockfile --locked ; then | |
| # echo "Cargo.lock for $d not up to date; please run ./scripts/ready-pull-request.sh and resubmit." | |
| # exit 1 | |
| # fi | |
| # fi | |
| popd | |
| done | |
| - name: Build and Test (Linux) | |
| if: runner.os == 'Linux' | |
| shell: bash -ex {0} | |
| run: | | |
| export RUST_BACKTRACE=1 | |
| # we need a basic git configuration for the tests to pass | |
| git config --global user.email [email protected] | |
| git config --global user.name "XetData Automation" | |
| pushd rust | |
| cargo test --verbose --no-fail-fast --features "strict openssl_vendored" | |
| popd | |
| pushd gitxet | |
| cargo test --verbose --no-fail-fast --features "openssl_vendored" | |
| popd | |
| pushd xetldfs | |
| cargo test --verbose --no-fail-fast --features "openssl_vendored" | |
| popd | |
| - name: Setup (macOS) | |
| if: runner.os == 'macOS' | |
| shell: bash -ex {0} | |
| run: | | |
| brew install openssl@3 | |
| # Upgrading make is required for cargo to play nicely with building openssl | |
| brew install make | |
| export PATH="/usr/local/opt/make/libexec/gnubin:$PATH" | |
| # Use our own shell to avoid arm64 vs arm64e issues | |
| brew install bash | |
| brew install md5sha1sum | |
| - name: Build and Test (macOS) | |
| if: runner.os == 'macOS' | |
| shell: /opt/homebrew/bin/bash -ex {0} | |
| run: | | |
| export PATH="/opt/homebrew/bin/:/usr/local/opt/make/libexec/gnubin:$PATH" | |
| # we need a basic git configuration for the tests to pass | |
| git config --global user.email [email protected] | |
| git config --global user.name "XetData Automation" | |
| pushd rust | |
| cargo test --verbose --no-fail-fast --features "strict openssl_vendored" | |
| popd | |
| pushd gitxet | |
| cargo test --verbose --no-fail-fast --features "openssl_vendored" | |
| popd | |
| # This is dumb. | |
| pushd xetldfs | |
| # cargo test --verbose --no-fail-fast --features "openssl_vendored" --target=x86_64-apple-darwin | |
| popd | |