Skip to content

Commit 9ec0e3e

Browse files
tboenigtboenig
committed
Update gtrepo.yml
1 parent 5cc4fad commit 9ec0e3e

File tree

1 file changed

+243
-79
lines changed

1 file changed

+243
-79
lines changed

‎.github/workflows/gtrepo.yml

+243-79
Original file line numberDiff line numberDiff line change
@@ -1,99 +1,263 @@
1-
name: gt-repo-scripts
1+
name: gtrepo
22
on:
33
push:
44
tags:
55
- 'v[0-9]+.[0-9]+.[0-9]+'
6+
67
workflow_dispatch:
8+
inputs:
9+
tag-name:
10+
description: Name of the release tag
11+
12+
defaults:
13+
run:
14+
shell: bash
15+
716
jobs:
8-
cli:
9-
name: gt-repo-scripts
10-
runs-on: ubuntu-latest
11-
steps:
17+
build:
18+
name: analyse and make Bagit
19+
runs-on: ubuntu-latest
20+
permissions:
21+
checks: write
22+
contents: write
23+
steps:
24+
- name: Checkout
25+
uses: actions/checkout@v4
1226

13-
- name: Git checkout
14-
uses: actions/checkout@v4
27+
- name: Using tag name from ref name
28+
if: github.event.inputs.tag-name == ''
29+
run: echo "TAG_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV
1530

31+
- name: Using tag name from input param
32+
if: github.event.inputs.tag-name != ''
33+
run: echo "TAG_NAME=${{ github.event.inputs.tag-name}}" >> $GITHUB_ENV
34+
35+
- name: download and install Saxon
36+
run: |
37+
wget https://github.com/Saxonica/Saxon-HE/releases/download/SaxonHE12-3/SaxonHE12-3J.zip
38+
unzip SaxonHE12-3J.zip
39+
rm SaxonHE12-3J.zip
1640
17-
41+
- name: install jq
42+
run: sudo apt-get install jq
43+
44+
- name: install XSL stylesheets
45+
run: |
46+
git clone https://github.com/tboenig/gt-repo-scripts.git
47+
mv gt-repo-scripts/scripts scripts/
48+
rm -r gt-repo-scripts
49+
50+
- name: install megalevelrules.xml
51+
run: |
52+
git clone --branch gh-pages --single-branch https://github.com/OCR-D/gt-MufiLevelRules.git
53+
mv gt-MufiLevelRules/rules/megalevelrules.xml scripts/megalevelrules.xml
54+
rm -r gt-MufiLevelRules
55+
56+
- name: convert metadata from YAML to JSON
57+
uses: mikefarah/yq@master
58+
with:
59+
cmd: yq -o=json METADATA.yml > METADATA.json
60+
61+
- name: check repo directory structure
62+
run: |
63+
mkdir ghout
64+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_unitTest.xsl \
65+
output=unitTest1 \
66+
-s:scripts/gt-overview_unitTest.xsl -o:ghout/pathtest.md
67+
68+
- name: test result
69+
run: |
70+
test -e ghout/pathtest.md
71+
if test -s ghout/pathtest.md; then \
72+
cat ghout/pathtest.md; false; fi
1873
19-
20-
21-
# Installation and Directories
74+
- name: install GT Labelling docs
75+
run: git clone https://github.com/tboenig/gt-guidelines.git
2276

23-
- name: install CITATION.cff update
24-
run: |
25-
git clone https://github.com/tboenig/CITATIONupdate.git
26-
77+
- name: make output directories
78+
run: mkdir metadata_out ocrdzip_out
2779

80+
- name: move README to readme_old/
81+
run: bash scripts/readmefolder.sh
2882

83+
- name: make readme.xml
84+
run: bash scripts/xreadme.sh
2985

30-
- name: transform yml to json
31-
uses: mikefarah/yq@master
32-
with:
33-
cmd: |
34-
yq -o=json CITATION.cff > CITATION.json
86+
- name: transform METADATA and make GT-Overview
87+
run: |
88+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
89+
output=METADATA repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY bagitDumpNum=$GITHUB_RUN_NUMBER releaseTag=${{ env.TAG_NAME }} \
90+
-s:scripts/gt-overview_metadata.xsl -o:ghout/metadata.md
3591
92+
- name: make compressed table view
93+
run: |
94+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
95+
output=TABLE repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY \
96+
-s:scripts/gt-overview_metadata.xsl -o:ghout/table.md
3697
37-
- name: Download and install saxon
38-
run: |
39-
wget https://github.com/Saxonica/Saxon-HE/releases/download/SaxonHE12-3/SaxonHE12-3J.zip
40-
unzip SaxonHE12-3J.zip
98+
- name: detailed table view
99+
run: |
100+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
101+
output=OVERVIEW repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY \
102+
-s:scripts/gt-overview_metadata.xsl -o:ghout/overview.md
41103
104+
- name: leveling the volume and documents
105+
run: |
106+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-level_parser.xsl \
107+
repoName=$GITHUB_REPOSITORY \
108+
-s:scripts/gt-level_parser.xsl -o:ghout/overview-level.md
42109
43-
44-
# Transformation and analysis
45-
46-
- name: update CITATION.cff
47-
run: |
48-
java -jar saxon-he-12.3.jar -xsl:CITATIONupdate/scripts/citationupdate.xsl \
49-
output=CITATION repoBase=$GITHUB_REF_Name repoName=$GITHUB_REPOSITORY bagitDumpNum=$GITHUB_RUN_NUMBER releaseTag=$GITHUB_REF_NAME \
50-
-s:CITATIONupdate/scripts/citationupdate.xsl -o:rawCITATION.cff
51-
shell: bash
52-
53-
54-
55-
- name: formating CITATION.cff
56-
uses: mikefarah/yq@master
57-
with:
58-
cmd: |
59-
yq -I4 rawCITATION.cff > CITATION.cff
60-
61-
62-
63-
- name: Create Upload GitHub release
64-
id: create-new-release
65-
uses: ncipollo/release-action@v1
66-
env:
67-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
68-
with:
69-
allowUpdates: true
70-
artifacts: 'gt-repo-scripts-package-v${{ github.run_number }}.zip'
71-
artifactContentType: application/zip
72-
tag: ${{ github.ref_name }}
73-
token: ${{ secrets.GITHUB_TOKEN }}
74-
name: gt-repo-scripts-package (Release${{ github.run_number }})
75-
omitNameDuringUpdate: true
76-
body: |
77-
<dl>
78-
<dt>Version:</dt>
79-
<dd>gt-repo-scripts-package (Release${{ github.run_number }}_${{ github.ref_name }})</dd>
80-
<dt>Info:</dt>
81-
<dd>
82-
The file gt-repo-scripts-package-v${{ github.run_number }}.zip is a zip archive file.<br/>
83-
<ul><li>If you wish to use the rules, this file must first be unpacked.</li>
84-
<li>The archive file contains several XSLT files and shell scripts.</li>
85-
<li>It is recommended to read the <a href="https://github.com/tboenig/gt-repo-scripts/blob/main/README.md">readme file</a> to understand the functionality and handling of the scripts.</li>
86-
<li>The XSLT files and shell scripts are optimized for implementation within a GitHub action workflow.</li></ul>
87-
</dd>
88-
</dl>
89-
</dl>
90-
110+
- name: generate mets.sh
111+
run: |
112+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
113+
output=METS repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY \
114+
-s:scripts/gt-overview_metadata.xsl -o:scripts/mets.sh
115+
ls -l scripts/mets.sh
116+
cat scripts/mets.sh
117+
118+
- name: generate Metadata JSON file
119+
run: |
120+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
121+
output=METAJSON repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY bagitDumpNum=$GITHUB_RUN_NUMBER releaseTag=${{ env.TAG_NAME }} \
122+
-s:scripts/gt-overview_metadata.xsl -o:metadata_out/metadata_l.json
123+
124+
- name: pretty-print JSON file
125+
run: |
126+
jq '.' metadata_out/metadata_l.json > metadata_out/metadata.json
127+
cp metadata_out/metadata.json ghout/
128+
rm metadata_out/metadata_l.json
129+
130+
- name: generate README
131+
run: |
132+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
133+
output=README repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY \
134+
-s:scripts/gt-overview_metadata.xsl -o:README.md
135+
136+
- name: generate METADATA_htr_united.yml
137+
run: |
138+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-metadata_htr_united.xsl repoName=${{ github.event.repository.name }} \
139+
-s:scripts/gt-metadata_htr_united.xsl
140+
141+
- name: generate METS Volume File
142+
run: |
143+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
144+
output=METSvolume repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY bagitDumpNum=$GITHUB_RUN_NUMBER releaseTag=${{ env.TAG_NAME }} \
145+
-s:scripts/gt-overview_metadata.xsl -o:metadata_out/mets.xml
146+
147+
- name: generate release download list
148+
run: |
149+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
150+
output=download repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY bagitDumpNum=$GITHUB_RUN_NUMBER releaseTag=${{ env.TAG_NAME }} \
151+
-s:scripts/gt-overview_metadata.xsl -o:ghout/download.txt
91152
153+
- name: delete fileGrp DEFAULT
154+
run: |
155+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
156+
output=METSdefault repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY bagitDumpNum=$GITHUB_RUN_NUMBER releaseTag=${{ env.TAG_NAME }} \
157+
-s:scripts/gt-overview_metadata.xsl
158+
159+
- name: generate CITATION.cff
160+
run: |
161+
java -jar saxon-he-12.3.jar -xsl:scripts/gt-overview_metadata.xsl \
162+
output=CITATION repoBase=${{ env.TAG_NAME }} repoName=$GITHUB_REPOSITORY bagitDumpNum=$GITHUB_RUN_NUMBER releaseTag=${{ env.TAG_NAME }} \
163+
-s:scripts/gt-overview_metadata.xsl -o:rawCITATION.cff
164+
165+
- name: pretty-print CITATION.cff
166+
uses: mikefarah/yq@master
167+
with:
168+
cmd: |
169+
yq -I4 rawCITATION.cff > CITATION.cff
170+
rm rawCITATION.cff
171+
172+
- name: symlink metadata as index
173+
run: ln -s ghout/metadata.md ghout/index.md
174+
175+
- name: ensure valid METS
176+
run: bash -ex scripts/data_mets.sh
92177

93-
- name: Commit CITATION.cff
94-
run: |
95-
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
96-
git config --local user.name "github-actions[bot]"
97-
git add CITATION.cff
98-
git commit -m "[Automatic] Update CITATION.cff files" || echo "Nothing to update"
99-
git push origin HEAD:main
178+
- name: install ocrd and bagit
179+
run: |
180+
sudo apt-get install -y python3 imagemagick libgeos-dev
181+
pip install -U pip 'setuptools>=61'
182+
pip install ocrd
183+
ocrd --version
184+
185+
- name: make validMets
186+
run: bash -ex scripts/mets.sh
187+
188+
- name: make bagit
189+
run: bash scripts/data_structure.sh
190+
191+
- name: copy CSS styles, Javascript and Markdown config files
192+
run: |
193+
cp scripts/table_hide.css ghout/
194+
cp scripts/levelparser.css ghout/
195+
cp scripts/lang.js ghout/
196+
cp scripts/_config.yml ghout/
197+
198+
- name: add metadata files to release assets
199+
uses: thedoctor0/zip-release@master
200+
with:
201+
filename: metadata-v${{ github.run_number }}.zip
202+
path: 'metadata_out'
203+
204+
- name: copy metadata.zip to ocrdzip_out
205+
run: cp metadata-v${{ github.run_number }}.zip ocrdzip_out/
206+
207+
- name: upload release assets
208+
uses: ncipollo/release-action@v1
209+
if: env.TAG_NAME != ''
210+
with:
211+
allowUpdates: true
212+
artifacts: 'ocrdzip_out/*.zip'
213+
artifactContentType: application/zip
214+
body: |
215+
<dl>
216+
<dt>Version:</dt>
217+
<dd>${{ env.TAG_NAME }}</dd>
218+
<dt>Info:</dt>
219+
<dd>
220+
To make use of Ground Truth, please download the provided zip files.<br/>
221+
The 'ocrd.zip' files are ocr-d-bagit files.<br/>
222+
The 'metadata-v${{ github.run_number }}.zip' file contains metadata for the Ground Truth corpus in both METS and JSON format.<br/>
223+
The 'mets.xml' file enumerates all the documents and BagIt files contained within.<br/>
224+
The bagits correspond to the <a href="https://ocr-d.de/de/spec/ocrd_zip.html">OCR-D Bagit Spec</a>.<br/>
225+
The source-code-zip and source-code-tar.gz files only provide metadata, citations, license and readme information.<br/>
226+
If you want to use the source files, please clone the repository.
227+
</dd>
228+
</dl>
229+
</dl>
230+
name: Release ${{ github.run_number }}_${{ env.TAG_NAME }}
231+
omitNameDuringUpdate: true
232+
tag: ${{ env.TAG_NAME }}
233+
token: ${{ secrets.GITHUB_TOKEN }}
234+
235+
- name: commit README
236+
run: |
237+
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
238+
git config --local user.name "github-actions[bot]"
239+
git add README.md
240+
git commit -m "[Automatic] Update readme files" || echo "Nothing to update"
241+
git push origin HEAD:main
242+
243+
- name: commit METADATA_htr_united.yml
244+
run: |
245+
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
246+
git config --local user.name "github-actions[bot]"
247+
git add ${{ github.event.repository.name }}_METADATA_htr_united.yml
248+
git commit -m "[Automatic] Update METADATA_htr_united.yml files" || echo "Nothing to update"
249+
git push origin HEAD:main
250+
251+
- name: commit CITATION.cff
252+
run: |
253+
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
254+
git config --local user.name "github-actions[bot]"
255+
git add CITATION.cff
256+
git commit -m "[Automatic] Update CITATION.cff files" || echo "Nothing to update"
257+
git push origin HEAD:main
258+
259+
- name: deploy GT Overview to GitHub Pages branch 🚀
260+
uses: JamesIves/github-pages-deploy-action@v4
261+
with:
262+
branch: gh-pages # The branch the action should deploy to.
263+
folder: ghout # The folder the action should deploy.

0 commit comments

Comments
 (0)