Skip to content

Commit c865ff3

Browse files
tgolssontgolsson
authored
Implement ark-legacy protocol (#10)
1 parent 552e84a commit c865ff3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+2480
-862
lines changed

Diff for: ‎.buildkite/pipeline.yml

+11-4
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ env:
1313
PY: "python3.9"
1414
PANTS_PYTHON_INTERPRETER_CONSTRAINTS: "['==3.9.*']"
1515
PANTS_CONFIG_FILES: "pants.ci.toml"
16+
PYTHON_BIN_NAME: "python3.9"
17+
PEX_PYTHON: "python3.9"
18+
1619

1720
steps:
1821
- group: ":passport_control: Validating PR"
@@ -32,27 +35,31 @@ steps:
3235
- label: ":package: Validating packages"
3336
command: |
3437
apt-get update && apt-get install -y swig
35-
bash ./pants package ::
38+
bash get-pants.sh
39+
/root/bin/pants -ldebug package ::
3640
3741
agents: *large
3842

3943
- label: ":package: Validating tailor"
4044
command: |
4145
apt-get update && apt-get install -y swig
42-
bash ./pants tailor --check ::
46+
bash get-pants.sh
47+
/root/bin/pants tailor --check ::
4348
4449
agents: *small
4550

4651
- label: ":python-black: :isort: Check-and-lint "
4752
command: |
4853
apt-get update && apt-get install -y swig
49-
bash ./pants update-build-files --check lint check ::
54+
bash get-pants.sh
55+
/root/bin/pants update-build-files --check lint check ::
5056
5157
agents: *small
5258

5359
- label: ":pytest: Run tests"
5460
command: |
5561
apt-get update && apt-get install -y swig
56-
bash ./pants test ::
62+
bash get-pants.sh
63+
/root/bin/pants test ::
5764
5865
agents: *small

Diff for: ‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ dist
1010
/logs/
1111
/runs/
1212
.pdm.toml
13+
/ark/

Diff for: ‎3rdparty/python/pytest.lock

+408
Large diffs are not rendered by default.

Diff for: ‎BUILD

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
python_requirements(
2-
name="requirements",
2+
name="reqs",
33
module_mapping={
44
"grpcio-reflection": ["grpc_reflection"],
55
"grpcio-health-checking": ["grpc_health"],
66
"google-cloud-storage": ["google.cloud"],
77
"google-auth": ["google.auth", "google.oauth2"],
88
"pyyaml": ["yaml"],
9-
"emote-rl": ["emote", "torch"],
9+
"emote-rl": ["emote"],
1010
},
1111
)
1212

1313
pex_binary(
1414
name="tensorboard",
1515
entry_point="tensorboard.main:run_main",
16-
dependencies=["//:requirements#tensorboard"],
16+
dependencies=["//:reqs#tensorboard"],
1717
)

Diff for: ‎CHANGELOG.md

-10
This file was deleted.

Diff for: ‎cmd/BUILD

+8-5
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,25 @@ python_sources()
33
python_source(
44
name="train",
55
source="train.py",
6-
dependencies=["//src/py:mock-dist-info", "//src/py/erupt-gym:package"],
6+
dependencies=["//src/py/utils/erupt.dist-info:mock-dist-info", "//src/py/erupt-gym:package"],
77
)
88

99
python_source(
1010
name="runner",
1111
source="runner.py",
12-
dependencies=["//src/py:mock-dist-info", "//src/py/erupt-gym:package"],
12+
dependencies=[
13+
"//src/py/utils/erupt.dist-info:mock-dist-info",
14+
],
1315
)
1416

15-
1617
python_source(
1718
name="server",
1819
source="server.py",
1920
dependencies=[
20-
"//src/py:mock-dist-info",
21+
"//src/py/utils/erupt.dist-info:mock-dist-info",
2122
"//src/py/erupt/erupt/entry_points/runner.py",
22-
"//src/py/erupt-gym:package",
23+
# "//src/py/erupt-ark/erupt_ark/engine.py",
24+
# "//src/py/erupt-gym/erupt_gym/engine.py"
2325
],
26+
# execution_mode="venv",
2427
)

Diff for: ‎cmd/runner.py

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
from erupt.entry_points import runner
2+
from erupt_ark.engine import Engine # noqa
3+
from erupt_gym.engine import GymEngine # noqa
24

35
runner.main()

Diff for: ‎cmd/server.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1-
from erupt.entry_points import server
1+
from erupt.entry_points import runner, server # noqa
2+
from erupt_ark.engine import Engine # noqa
3+
from erupt_gym.engine import GymEngine # noqa
24

35
server.main()

Diff for: ‎docs/src/recipes/ipython.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ Looking at for example `//cmd:train`, it is (as of writing) defined like this:
5757
python_source(
5858
name="train",
5959
source="train.py",
60-
dependencies=["//src/py:mock-dist-info", "//src/py/erupt-gym:package"],
60+
dependencies=["//src/py/utils/erupt.dist-info:mock-dist-info", "//src/py/erupt-gym:package"],
6161
)
6262
```
6363
```python

Diff for: ‎get-pants.sh

+221
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
3+
# Licensed under the Apache License, Version 2.0 (see LICENSE).
4+
5+
set -euo pipefail
6+
7+
COLOR_RED="\x1b[31m"
8+
COLOR_GREEN="\x1b[32m"
9+
COLOR_YELLOW="\x1b[33m"
10+
COLOR_RESET="\x1b[0m"
11+
12+
function log() {
13+
echo -e "$@" 1>&2
14+
}
15+
16+
function die() {
17+
(($# > 0)) && log "${COLOR_RED}$*${COLOR_RESET}"
18+
exit 1
19+
}
20+
21+
function green() {
22+
(($# > 0)) && log "${COLOR_GREEN}$*${COLOR_RESET}"
23+
}
24+
25+
function warn() {
26+
(($# > 0)) && log "${COLOR_YELLOW}$*${COLOR_RESET}"
27+
}
28+
29+
function check_cmd() {
30+
local cmd="$1"
31+
command -v "$cmd" > /dev/null || die "This script requires the ${cmd} binary to be on the PATH."
32+
}
33+
34+
help_url="https://www.pantsbuild.org/docs/getting-help"
35+
36+
_GC=()
37+
38+
function gc() {
39+
if (($# > 0)); then
40+
check_cmd rm
41+
_GC+=("$@")
42+
else
43+
rm -rf "${_GC[@]}"
44+
fi
45+
}
46+
47+
trap gc EXIT
48+
49+
check_cmd uname
50+
51+
function calculate_os() {
52+
local os
53+
54+
os="$(uname -s)"
55+
if [[ "${os}" =~ [Ll]inux ]]; then
56+
echo linux
57+
elif [[ "${os}" =~ [Dd]arwin ]]; then
58+
echo macos
59+
elif [[ "${os}" =~ [Ww]in|[Mm][Ii][Nn][Gg] ]]; then
60+
# Powershell reports something like: Windows_NT
61+
# Git bash reports something like: MINGW64_NT-10.0-22621
62+
echo windows
63+
else
64+
die "Pants is not supported on this operating system (${os}). Please reach out to us at ${help_url} for help."
65+
fi
66+
}
67+
68+
OS="$(calculate_os)"
69+
70+
check_cmd basename
71+
if [[ "${OS}" == "windows" ]]; then
72+
check_cmd pwsh
73+
else
74+
check_cmd curl
75+
fi
76+
77+
function fetch() {
78+
local url="$1"
79+
local dest_dir="$2"
80+
81+
local dest
82+
dest="${dest_dir}/$(basename "${url}")"
83+
84+
if [[ "${OS}" == "windows" ]]; then
85+
pwsh -c "Invoke-WebRequest -OutFile $dest -Uri $url"
86+
else
87+
curl --proto '=https' --tlsv1.2 -sSfL -o "${dest}" "${url}"
88+
fi
89+
}
90+
91+
if [[ "${OS}" == "macos" ]]; then
92+
check_cmd shasum
93+
else
94+
check_cmd sha256sum
95+
fi
96+
97+
function sha256() {
98+
if [[ "${OS}" == "macos" ]]; then
99+
shasum --algorithm 256 "$@"
100+
else
101+
sha256sum "$@"
102+
fi
103+
}
104+
105+
check_cmd mktemp
106+
107+
function install_from_url() {
108+
local url="$1"
109+
local dest="$2"
110+
111+
local workdir
112+
workdir="$(mktemp -d)"
113+
gc workdir
114+
115+
fetch "${url}.sha256" "${workdir}"
116+
fetch "${url}" "${workdir}"
117+
(
118+
cd "${workdir}"
119+
sha256 -c --status ./*.sha256 ||
120+
die "Download from ${url} did not match the fingerprint at ${url}.sha256"
121+
)
122+
rm "${workdir}/"*.sha256
123+
if [[ "${OS}" == "macos" ]]; then
124+
mkdir -p "$(dirname "${dest}")"
125+
install -m 755 "${workdir}/"* "${dest}"
126+
else
127+
install -D -m 755 "${workdir}/"* "${dest}"
128+
fi
129+
}
130+
131+
function calculate_arch() {
132+
local arch
133+
134+
arch="$(uname -m)"
135+
if [[ "${arch}" =~ x86[_-]64 ]]; then
136+
echo x86_64
137+
elif [[ "${arch}" =~ arm64|aarch64 ]]; then
138+
echo aarch64
139+
else
140+
die "Pants is not supported for this chip architecture (${arch}). Please reach out to us at ${help_url} for help."
141+
fi
142+
}
143+
144+
check_cmd cat
145+
146+
function usage() {
147+
cat << EOF
148+
Usage: $0
149+
150+
Installs the pants launcher binary.
151+
152+
You only need to run this once on a machine when you do not have "pants"
153+
available to run yet.
154+
155+
The pants binary takes care of managing and running the underlying
156+
Pants version configured in "pants.toml" in the surrounding Pants-using
157+
project.
158+
159+
Once installed, if you want to update your "pants" launcher binary, use
160+
"SCIE_BOOT=update pants" to get the latest release or
161+
"SCIE_BOOT=update pants --help" to learn more options.
162+
163+
-h | --help: Print this help message.
164+
165+
-d | --bin-dir:
166+
The directory to install the scie-pants binary in, "~/bin" by default.
167+
168+
-b | --base-name:
169+
The name to use for the scie-pants binary, "pants" by default.
170+
171+
-V | --version:
172+
The version of the scie-pants binary to install, the latest version by default.
173+
The available versions can be seen at:
174+
https://github.com/pantsbuild/scie-pants/releases
175+
176+
EOF
177+
}
178+
179+
bin_dir="${HOME}/bin"
180+
base_name="pants"
181+
version="latest/download"
182+
while (($# > 0)); do
183+
case "$1" in
184+
--help | -h)
185+
usage
186+
exit 0
187+
;;
188+
--bin-dir | -d)
189+
bin_dir="$2"
190+
shift
191+
;;
192+
--base-name | -b)
193+
base_name="$2"
194+
shift
195+
;;
196+
--version | -V)
197+
version="download/v$2"
198+
shift
199+
;;
200+
*)
201+
usage
202+
die "Unexpected argument $1\n"
203+
;;
204+
esac
205+
shift
206+
done
207+
208+
ARCH="$(calculate_arch)"
209+
URL="https://github.com/pantsbuild/scie-pants/releases/${version}/scie-pants-${OS}-${ARCH}"
210+
dest="${bin_dir}/${base_name}"
211+
212+
log "Downloading and installing the pants launcher ..."
213+
install_from_url "${URL}" "${dest}"
214+
green "Installed the pants launcher from ${URL} to ${dest}"
215+
if ! command -v "${base_name}" > /dev/null; then
216+
warn "${dest} is not on the PATH."
217+
log "You'll either need to invoke ${dest} explicitly or else add ${bin_dir} to your shell's PATH."
218+
fi
219+
220+
green "\nRunning \`pants\` in a Pants-enabled repo will use the version of Pants configured for that repo."
221+
green "In a repo not yet Pants-enabled, it will prompt you to set up Pants for that repo."

0 commit comments

Comments
 (0)