Skip to content

Commit 7cafc84

Browse files
rrelledgerrelledge
committed
Merge branch 'main' into release-rs-gilboa
2 parents acf6e8b + 0a4da35 commit 7cafc84

File tree

3,954 files changed

+232973
-74046
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

3,954 files changed

+232973
-74046
lines changed

‎.github/workflows/autocomment.yaml

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,10 @@ jobs:
4949
-H "X-GitHub-Api-Version: 2022-11-28" \
5050
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
5151
$FILES_URL \
52-
| jq -r --arg prefix $BRANCH_NAME/ '.[] | select(((.filename | test("content\/.+\\.md")) and .status != "removed")) | ($prefix + .filename)' \
52+
| jq -r --arg prefix $BRANCH_NAME/ '.[] | select(((.filename | test("content/(?!.*embed).*\\.md")) and .status != "removed")) | ($prefix + .filename)' \
5353
| sed -E -e 's|(^[^/]+/)([^/]+/)|\1|' -e 's|^|https://redis.io/docs/staging/|' -e 's|(_?index)?\.md||' \
54-
| uniq \
55-
| xargs \
56-
| sed 's/ /<br>/g')
54+
| sort \
55+
| uniq)
5756
5857
# Get all changed image files
5958
CHANGED_IMAGE_FILES=$(curl -Ls \
@@ -63,23 +62,51 @@ jobs:
6362
$FILES_URL \
6463
| jq -r '.[] | select(.filename | test("^static/images\/.+(.png|.svg|.gif|.)")) | .filename | sub("^static/";"")')
6564
65+
# Get all changed embeds files
66+
CHANGED_EMBED_FILES=$(curl -Ls \
67+
-H "Accept: application/vnd.github+json" \
68+
-H "X-GitHub-Api-Version: 2022-11-28" \
69+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
70+
$FILES_URL \
71+
| jq -r '.[] | select(.filename | test("^content/embeds\/.+.md")) | .filename | split("/")[-1]')
72+
6673
if [[ -n "$CHANGED_IMAGE_FILES" ]]
6774
then
6875
# For each image, work out in which README it is present
6976
MD_FILES_WITH_IMAGE=()
70-
for CHANGED_IMAGE_FILE in $CHANGED_IMAGE_FILES; do
77+
for CHANGED_IMAGE_FILE in "${CHANGED_IMAGE_FILES[@]}"; do
7178
MD_FILE_WITH_IMAGE=$(grep -ro "$CHANGED_IMAGE_FILE" content \
7279
| sed -E -e 's|:.+||' -e "s|^content/|https://redis.io/docs/staging/$BRANCH_NAME/|" -e 's|(_?index)?\.md||' \
80+
| grep -v "https://redis.io/docs/staging/$BRANCH_NAME/embeds/" \
81+
| sort \
7382
| uniq)
7483
MD_FILES_WITH_IMAGE+=($MD_FILE_WITH_IMAGE)
7584
done
76-
CHANGED_MD_FILES_WITH_IMAGE=$(printf "%s\n" "${MD_FILES_WITH_IMAGE[@]}" \
77-
| uniq \
78-
| xargs \
79-
| sed 's/ /<br>/g')
80-
CHANGED_MD_FILES="${CHANGED_MD_FILES}<br>${CHANGED_MD_FILES_WITH_IMAGE}"
85+
CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" "${MD_FILES_WITH_IMAGE[@]}" \
86+
| sort \
87+
| uniq)
88+
fi
89+
90+
if [[ -n "$CHANGED_EMBED_FILES" ]]
91+
then
92+
# For each embed, work out in which README it is present
93+
MD_FILES_WITH_EMBED=()
94+
for CHANGED_EMBED_FILE in "${CHANGED_EMBED_FILES[@]}"; do
95+
MD_FILE_WITH_EMBED=$(grep -ro "< embed-md \"${CHANGED_EMBED_FILE}\" >" content \
96+
| sed -E -e 's|:.+||' -e "s|^content/|https://redis.io/docs/staging/$BRANCH_NAME/|" -e 's|(_?index)?\.md||' \
97+
| sort \
98+
| uniq)
99+
MD_FILES_WITH_EMBED+=($MD_FILE_WITH_EMBED)
100+
done
101+
CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" "${MD_FILES_WITH_EMBED[@]}" \
102+
| sort \
103+
| uniq)
81104
fi
82105
106+
CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" \
107+
| xargs \
108+
| sed 's/ /<br>/g')
109+
83110
if [[ -z "$CHANGED_MD_FILES" ]]
84111
then
85112
if [[ -z "$EXISTING_COMMENT_ID" ]]

‎.github/workflows/cleanup_pr_folders.yaml

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: cleanup_pr_folders
2+
3+
on:
4+
pull_request:
5+
types:
6+
- closed
7+
8+
jobs:
9+
staging:
10+
if: github.event.pull_request.merged == true
11+
runs-on: ubuntu-latest
12+
permissions:
13+
contents: 'read'
14+
id-token: 'write'
15+
env:
16+
STAGING_BUCKET: docs-staging-learn-redis-com
17+
STAGING_PROJECT_ID: ${{ secrets.GCP_PROJECT_STAGING }}
18+
STAGING_SERVICE_ACCOUNT: ${{ secrets.STAGING_SERVICE_ACCOUNT }}
19+
STAGING_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.STAGING_WORKLOAD_IDENTITY_PROVIDER }}
20+
21+
steps:
22+
- name: 'Google auth'
23+
uses: 'google-github-actions/auth@v2'
24+
with:
25+
project_id: '${{ env.STAGING_PROJECT_ID }}'
26+
service_account: '${{ env.STAGING_SERVICE_ACCOUNT }}'
27+
workload_identity_provider: '${{ env.STAGING_WORKLOAD_IDENTITY_PROVIDER }}'
28+
29+
- name: 'Set up Cloud SDK'
30+
uses: 'google-github-actions/setup-gcloud@v2'
31+
with:
32+
project_id: '${{ env.STAGING_PROJECT_ID }}'
33+
version: '>= 363.0.0'
34+
35+
- name: Remove pull request staging folder
36+
run: |
37+
BRANCH_TO_CLEANUP="${{ github.event.pull_request.head.ref }}"
38+
STAGING_BUCKET="${{ env.STAGING_BUCKET }}"
39+
40+
if gsutil -q stat gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*; then
41+
gsutil -q -m rm -r gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*
42+
echo "Removed gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}"
43+
fi
44+
45+
prod:
46+
if: github.event.pull_request.merged == true
47+
runs-on: ubuntu-latest
48+
permissions:
49+
contents: 'read'
50+
id-token: 'write'
51+
env:
52+
PROD_BUCKET: docs-prod-learn-redis-com
53+
PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }}
54+
PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }}
55+
PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }}
56+
57+
steps:
58+
- name: 'Google auth'
59+
uses: 'google-github-actions/auth@v2'
60+
with:
61+
project_id: '${{ env.PROD_PROJECT_ID }}'
62+
service_account: '${{ env.PROD_SERVICE_ACCOUNT }}'
63+
workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}'
64+
65+
- name: 'Set up Cloud SDK'
66+
uses: 'google-github-actions/setup-gcloud@v2'
67+
with:
68+
project_id: '${{ env.PROD_PROJECT_ID }}'
69+
version: '>= 363.0.0'
70+
71+
- name: Remove pull request staging folder
72+
run: |
73+
BRANCH_TO_CLEANUP="${{ github.event.pull_request.head.ref }}"
74+
PROD_BUCKET="${{ env.PROD_BUCKET }}"
75+
76+
if gsutil -q stat gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*; then
77+
gsutil -q -m rm -r gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*
78+
echo "Removed gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}"
79+
fi

‎.github/workflows/main-staging.yml

Lines changed: 74 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,17 @@ jobs:
66
build_docs:
77
name: Build the staging documentation site
88
runs-on: ubuntu-latest
9+
permissions:
10+
contents: 'read'
11+
id-token: 'write'
912
env:
10-
HUGO_VERSION: 0.111.2
13+
HUGO_VERSION: 0.143.1
1114
GCLOUD_VERSION: 458.0.1-linux-x86_64
1215
BUCKET: docs-staging-learn-redis-com
13-
BUCKET_SERVICE_ACCOUNT: ${{ secrets.BUCKET_DOCUMENTATION_SA_STAGING }}
14-
BUCKET_SECRET: ${{ secrets.BUCKET_DOCUMENTATION_STAGING }}
15-
GCP_PROJECT: ${{ secrets.GCP_PROJECT_STAGING }}
16+
STAGING_PROJECT_ID: ${{ secrets.GCP_PROJECT_STAGING }}
17+
STAGING_SERVICE_ACCOUNT: ${{ secrets.STAGING_SERVICE_ACCOUNT }}
18+
STAGING_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.STAGING_WORKLOAD_IDENTITY_PROVIDER }}
19+
1620
steps:
1721
- name: Start
1822
run: echo "The name of the branch is ${{ github.ref }} on ${{ github.repository }}"
@@ -60,6 +64,7 @@ jobs:
6064
setBaseUrl
6165
kubernetes_versions=($(find content/operate/kubernetes/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}'))
6266
rs_versions=($(find content/operate/rs/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}'))
67+
rdi_versions=($(find content/integrate/redis-data-integration/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}'))
6368
6469
# build latest
6570
for version in "${kubernetes_versions[@]}"; do
@@ -68,6 +73,9 @@ jobs:
6873
for version in "${rs_versions[@]}"; do
6974
rm -r "content/operate/rs/${version}"
7075
done
76+
for version in "${rdi_versions[@]}"; do
77+
rm -r "content/integrate/redis-data-integration/${version}"
78+
done
7179
make all
7280
7381
git checkout .
@@ -83,13 +91,17 @@ jobs:
8391
rm -r "content/operate/kubernetes/${version_to_remove}"
8492
done
8593
86-
cp -r "content/operate/kubernetes/${version}"/* content/operate/kubernetes/
87-
rm -r "content/operate/kubernetes/${version}"
94+
# relrefs should not include version
95+
find "content/operate/kubernetes/${version}" -type f | while read file; do
96+
awk '{gsub(/\(\{\{< ?relref "\/operate\/kubernetes\/'${version}'/, "({{< relref \"/operate/kubernetes") }1' "$file" > tmpfile && mv tmpfile "$file"
97+
done
98+
99+
rsync -a --delete-after "content/operate/kubernetes/${version}"/ content/operate/kubernetes/
88100
sed -i 's/id="versionSelectorKubernetesValue" class="version-selector-control">latest/id="versionSelectorKubernetesValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html
89101
sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis for Kubernetes/' content/operate/kubernetes/_index.md
90102
91103
# inject replace command in meta-links to make sure editURL and issuesURL point to right version
92-
sed -i "11i \{\{ \$gh_file = replaceRE \`\^operate\/kubernetes\/\` \"operate\/kubernetes\/$version\/\" \$gh_file }}" layouts/partials/meta-links.html
104+
sed -i "12i \{\{ \$gh_path = replaceRE \`\^operate\/kubernetes\/\` \"operate\/kubernetes\/$version\/\" \$gh_path }}" layouts/partials/meta-links.html
93105
94106
hugo -d "kubernetes-${version}"
95107
@@ -106,34 +118,68 @@ jobs:
106118
rm -r "content/operate/rs/${version_to_remove}"
107119
done
108120
109-
cp -r "content/operate/rs/${version}"/* content/operate/rs/
110-
rm -r "content/operate/rs/${version}"
121+
# relrefs should not include version
122+
find "content/operate/rs/${version}" -type f | while read file; do
123+
awk '{gsub(/\(\{\{< ?relref "\/operate\/rs\/'${version}'/, "({{< relref \"/operate/rs") }1' "$file" > tmpfile && mv tmpfile "$file"
124+
done
125+
126+
rsync -a --delete-after "content/operate/rs/${version}"/ content/operate/rs/
111127
sed -i 's/id="versionSelectorRsValue" class="version-selector-control">latest/id="versionSelectorRsValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html
112128
sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis Software/' content/operate/rs/_index.md
113129
114130
# inject replace command in meta-links to make sure editURL and issuesURL point to right version
115-
sed -i "11i \{\{ \$gh_file = replaceRE \`\^operate\/rs\/\` \"operate\/rs\/$version\/\" \$gh_file }}" layouts/partials/meta-links.html
131+
sed -i "12i \{\{ \$gh_path = replaceRE \`\^operate\/rs\/\` \"operate\/rs\/$version\/\" \$gh_path }}" layouts/partials/meta-links.html
116132
117133
hugo -d "rs-${version}"
118134
119135
git checkout .
120136
done
121137
138+
for version in "${rdi_versions[@]}"; do
139+
140+
setBaseUrl
141+
142+
# for each version, remove all other versions before building
143+
versions_to_remove=($(echo "${rdi_versions[@]}" "${version}" | tr ' ' '\n' | sort | uniq -u))
144+
for version_to_remove in "${versions_to_remove[@]}"; do
145+
rm -r "content/integrate/redis-data-integration/${version_to_remove}"
146+
done
147+
148+
# relrefs should not include version
149+
find "content/operate/redis-data-integration/${version}" -type f | while read file; do
150+
awk '{gsub(/\(\{\{< ?relref "\/operate\/redis-data-integration\/'${version}'/, "({{< relref \"/operate/redis-data-integration") }1' "$file" > tmpfile && mv tmpfile "$file"
151+
done
152+
153+
rsync -a --delete-after "content/operate/redis-data-integration/${version}"/ content/operate/redis-data-integration/
154+
sed -i 's/id="versionSelectorRedis-Data-IntegrationValue" class="version-selector-control">latest/id="versionSelectorRedis-Data-IntegrationValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html
155+
sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis Data Integration/' content/integrate/redis-data-integration/_index.md
156+
157+
# inject replace command in meta-links to make sure editURL and issuesURL point to right version
158+
sed -i "12i \{\{ \$gh_path = replaceRE \`\^integrate\/redis-data-integration\/\` \"integrate\/redis-data-integration\/$version\/\" \$gh_path }}" layouts/partials/meta-links.html
159+
160+
hugo -d "redis-data-integration-${version}"
161+
162+
git checkout .
163+
done
164+
122165
- name: List client examples
123166
run: ls "${{ github.workspace }}/examples"
124167
- name: List files to be published
125168
run: ls "${{ github.workspace }}/public"
126-
- name: Install the Google Cloud CLI
127-
run: |
128-
wget -O ${{ github.workspace }}/google-cloud-cli.tar.gz "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-$GCLOUD_VERSION.tar.gz" \
129-
&& tar -xvf google-cloud-cli.tar.gz -C ${{ github.workspace }}\
130-
&& ${{ github.workspace }}/google-cloud-sdk/install.sh --quiet
131-
- name: Prepare bucket authentication
132-
run: echo $BUCKET_SECRET | base64 --decode > service_account.json && ls -a service_account.json
133-
- name: Authenticate to the bucket
134-
run: |
135-
./google-cloud-sdk/bin/gcloud auth activate-service-account $BUCKET_SERVICE_ACCOUNT --key-file=./service_account.json --project=$GCP_PROJECT \
136-
&& ./google-cloud-sdk/bin/gcloud auth list
169+
170+
- name: 'Google auth'
171+
uses: 'google-github-actions/auth@v2'
172+
with:
173+
project_id: '${{ env.STAGING_PROJECT_ID }}'
174+
service_account: '${{ env.STAGING_SERVICE_ACCOUNT }}'
175+
workload_identity_provider: '${{ env.STAGING_WORKLOAD_IDENTITY_PROVIDER }}'
176+
177+
- name: 'Set up Cloud SDK'
178+
uses: 'google-github-actions/setup-gcloud@v2'
179+
with:
180+
project_id: '${{ env.STAGING_PROJECT_ID }}'
181+
version: '>= 363.0.0'
182+
137183
- name: Sync the branch to the bucket
138184
run: |
139185
if [[ "${{ github.ref_name }}" == "main" ]]
@@ -148,12 +194,16 @@ jobs:
148194
else
149195
bucket_path=staging/${{ github.ref_name }}
150196
fi \
151-
&& ./google-cloud-sdk/bin/gsutil -m rsync -r -c -j html -d ${{ github.workspace }}/public gs://$BUCKET/$bucket_path
197+
&& gsutil -m rsync -r -c -j html -d ${{ github.workspace }}/public gs://$BUCKET/$bucket_path
152198
153199
versioned_builds=($(find . -type d -regex ".*[0-9-]" -maxdepth 1 | sed -E 's/^.\///'))
154200
for versioned_build in "${versioned_builds[@]}"; do
155-
product=$(grep -Eo '[a-zA-Z]+' <<< $versioned_build)
156-
gsutil -m rsync -r -c -j html -d "${{ github.workspace }}/${versioned_build}/operate/${product}" "gs://${BUCKET}/${versioned_build}/operate/${product}"
201+
product=$(awk 'BEGIN{FS=OFS="-"}{NF--; print}' <<< $versioned_build)
202+
if [[ "${product}" == "redis-data-integration" ]]; then
203+
gsutil -m rsync -r -c -j html -d "${{ github.workspace }}/${versioned_build}/integrate/${product}/" "gs://${BUCKET}/${bucket_path}/${versioned_build}/integrate/${product}"
204+
else
205+
gsutil -m rsync -r -c -j html -d "${{ github.workspace }}/${versioned_build}/operate/${product}/" "gs://${BUCKET}/${bucket_path}/${versioned_build}/operate/${product}"
206+
fi
157207
done
158208
159209
- name: End

0 commit comments

Comments
 (0)