Skip to content

Commit 34ee2fa

Browse files
hboutemyhboutemy
authored
Merge pull request #5 from sonatype-nexus-community/script-main
maintain update script in main branch
2 parents dd0274d + 3254c12 commit 34ee2fa

File tree

3 files changed

+114
-1
lines changed

3 files changed

+114
-1
lines changed

‎.github/workflows/update.yml

+7-1
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,14 @@ jobs:
1111
build:
1212
runs-on: ubuntu-latest
1313
steps:
14-
- uses: actions/checkout@v4
14+
- name: Checkout
15+
uses: actions/checkout@v4
16+
17+
- name: Checkout gh-pages
18+
uses: actions/checkout@v4
1519
with:
1620
ref: 'gh-pages'
21+
path: 'gh-pages'
1722

1823
- name: Update Reports
1924
run: ./update-pytorch.sh
@@ -22,6 +27,7 @@ jobs:
2227
env:
2328
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2429
run: |
30+
cd gh-pages
2531
git diff-index --quiet HEAD && exit 0 # commit only if something to do
2632
git config --local user.email "[email protected]"
2733
git config --local user.name "Update CI"

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
gh-pages/

‎update-pytorch.sh

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/bin/bash
2+
3+
[ -d gh-pages ] || git clone -q --branch gh-pages $(git config --get remote.origin.url) gh-pages
4+
cd gh-pages
5+
6+
# update a PyPI index from PyTorch = https://download.pytorch.org/$d with d=whl or whl/<compute platform>
7+
# this copies the content (main url + follows links to projects) and does 2 updates:
8+
# 1. copies the content in a "simple/" sub-directory to match the convention from PEP 503 simple (that allows other APIs in parallel)
9+
# 2. updates the links to binary packages by adding "https://download.pytorch.org" prefix to PyTorch-provided "/whl/*" path to link back to PyTorch binaries in their home location
10+
function updateIndex() {
11+
local d=$1
12+
13+
# will copy source $d to $d/simple
14+
mkdir -p $d/simple
15+
local dir="$(pwd)"
16+
cd $d/simple
17+
18+
# projects list
19+
curl -s https://download.pytorch.org/$d/ | grep -v 'TIMESTAMP 1' > index.html
20+
local count="$(cat index.html | cut -d '>' -f 2 | cut -d '<' -f 1 | grep -cve '^\s*$')"
21+
22+
echo "https://download.pytorch.org/$d/ => $d/simple/"
23+
checkCount $d $count 40
24+
25+
# copy also content of each project
26+
local i=0
27+
for p in `cat index.html | cut -d '>' -f 2 | cut -d '<' -f 1`
28+
do
29+
mkdir -p $p
30+
cd $p
31+
((i++))
32+
curl -s https://download.pytorch.org/$d/$p/ \
33+
| sed -e 's_href="/whl_href="https://download.pytorch.org/whl_' \
34+
| grep -v 'TIMESTAMP 1' \
35+
> index.html
36+
37+
local pcount="$(cat index.html | grep -c 'https://download.pytorch.org/whl/')"
38+
printf "%5d / $count $d/$p/ => $d/simple/$p/ $pcount\n" $i
39+
checkCount $d/$p $count 1
40+
cd ..
41+
done
42+
echo
43+
cd "$dir"
44+
}
45+
46+
function checkCount() {
47+
local content="$1"
48+
local count="$2"
49+
local minimum="$3"
50+
51+
if [ $count -lt $minimum ]
52+
then
53+
echo "!!! failing because low packages count for $content: $count (probably intermittent download failure)"
54+
exit 1
55+
fi
56+
}
57+
58+
# update main PyTorch index, that contains everything, whatever the compute platform
59+
updateIndex "whl"
60+
61+
# see resulting updates
62+
git update-index -q --refresh
63+
git diff-index --name-status HEAD
64+
65+
if `git diff-index --quiet HEAD`
66+
then
67+
echo "no update found in PyTorch main index."
68+
exit 0
69+
fi
70+
echo "updates found in PyTorch main index: updating also compute platform specific ones..."
71+
72+
# update compute-platform specific indexes
73+
# ignore old non-updated ones:
74+
# - cu75 cu80 cu90 cu91 cu92 cu100 cu101 cu102 cu110 cu111 cu113 cu115 cu116 cu117 cu117_pypi_cudnn
75+
# - rocm3.10 rocm3.7 rocm3.8 rocm4.0.1 rocm4.1 rocm4.2 rocm4.3.1 rocm4.5.2 rocm5.0 rocm5.1.1 rocm5.2 rocm5.3 rocm5.4.2 rocm5.5 rocm5.6 rocm5.7
76+
for d in cpu cpu-cxx11-abi cpu_pypi_pkg cu118 cu121 cu124 rocm6.0 rocm6.1
77+
do
78+
updateIndex "whl/$d"
79+
done
80+
81+
du -sh whl/*
82+
83+
for d in whl/simple whl/*/simple ; do echo "$(ls $d | wc -l | xargs) $d" ; done > summary.txt
84+
cat summary.txt
85+
86+
cat <<EOF > whl/index.html
87+
<!DOCTYPE html>
88+
<html>
89+
<body>
90+
<h1>Sonatype <a href="https://pytorch.org/">PyTorch</a> PyPI <b>improved</b> indexes</h1>
91+
92+
Generated by <a href="https://github.com/sonatype-nexus-community/pytorch-pypi">sonatype-nexus-community/pytorch-pypi</a> from <a href="https://download.pytorch.org/whl/">https://download.pytorch.org/whl/</a>
93+
94+
<ul>
95+
<li><a href="simple">simple</a> full index</li>
96+
<li>compute platform filtered indexes:<ul>
97+
$(for d in whl/*
98+
do
99+
d="$(echo $d | cut -c 5-)"
100+
[ -d "whl/$d" ] && [ "$d" != "simple" ] && echo "<li><a href=\"$d/simple\">$d/simple</a></li>"
101+
done)
102+
</ul></li>
103+
</ul>
104+
</body>
105+
</html>
106+
EOF

0 commit comments

Comments
 (0)