build

#!/bin/bash -e

# Run:
#   docker run --rm -it --user $(id -u):$(id -g) -v $PWD:/documents/ gbif/asciidoctor-toolkit:latest

# Enable the **/*.$PRIMARY_LANGUAGE.adoc below.
shopt -s globstar
# In case there are no translations.
shopt -s nullglob

if [[ -t 0 ]]; then
	RED='\033[1;31m'
	NC='\033[0m'
else
	RED='▶▶▶ '
	NC=''
fi

echo -e "${RED}GBIF Asciidoctor Toolkit v${GBIF_ASCIIDOCTOR_TOOLKIT_VERSION}.${NC}"
echo

OPTS=$(getopt --options 'h' --long 'help,no-compress,no-git,no-pdf,no-spellcheck,no-translation,no-video,presentation,url:' -n $0 -- "$@")

if [[ $? -ne 0 ]]; then
	echo 'Invalid arguments.' >&2
	echo "Usage:" >&2
	echo "$0 [--no-compress] [--no-git] [--no-pdf] [--no-spellcheck] [--no-translation] [--no-video] [--presentation] [--url]" >&2
	exit 1
fi

eval set -- "$OPTS"
unset OPTS

while true; do
	case "$1" in
		'--no-compress')
			no_compress=1
			shift 1
			continue
			;;
		'--no-git')
			no_git=1
			shift 1
			continue
			;;
		'--no-pdf')
			no_pdf=1
			shift 1
			continue
			;;
		'--no-spellcheck')
			no_spellcheck=1
			shift 1
			continue
			;;
		'--no-translation')
			no_translation=1
			shift 1
			continue
			;;
		'--no-video')
			no_video=1
			shift 1
			continue
			;;
		'--presentation')
			presentation=1
			shift 1
			continue
			;;
		'--url')
			url=$2
			shift 2
			continue
			;;
		'-h'|'--help')
			echo "Usage:"
			echo "$0 [--no-compress] [--no-git] [--presentation] [--no-pdf] [--no-spellcheck] [--no-translation] [--no-video]"
			exit 0
			;;
		'--')
			shift
			break
			;;
		*)
			echo "Internal error!" >&2
			exit 1
			;;
	esac
done

export PARALLEL_HOME=/tmp

# Check this is run from a reasonable directory.
# (Advanced users can run /bin/bash as a Docker target command.)
if [[ -z $no_git ]] && ! git ls-files --error-unmatch 'index.??.adoc' > /dev/null; then
	echo >&2 "There is no index.en.adoc file in this directory."
	echo >&2
	echo >&2 "Check you are running this from the top-level of your document,"
	echo >&2 "and that there is a source document called index.en.adoc."
	echo >&2
	echo >&2 "(Substitute en with another two-letter language code as required.)"
	exit 1
fi

if [[ -z $no_git ]]; then
	PRIMARY_LANGUAGE=$(git ls-files 'index.??.adoc' | cut -d. -f 2)
else
	PRIMARY_LANGUAGE=en
fi

echo
echo -e "${RED}Primary language is $PRIMARY_LANGUAGE.${NC}"

# Clean any old files
if [[ -z $no_git ]]; then
	echo "Removing files ignored by Git"
	git clean -f -X

	# Set timestamp on index documents to latest commit time (shown in document footer)
	for i in index.$PRIMARY_LANGUAGE.adoc translations/index.??.adoc translations/index.??-??.adoc translations/index.??-???.adoc; do
		touch --no-create --date @"$(git rev-list --max-count=1 --pretty=%ct HEAD | tail -n 1)" $i
	done
	echo
fi

# Produce the translated adoc source from the po-files.
if [[ -z $no_translation ]] && [[ -e po4a.conf ]]; then
	echo -e "${RED}Translating sources${NC}"
	for pri_adoc in *.$PRIMARY_LANGUAGE.adoc; do
		grep -q "$pri_adoc" po4a.conf || echo >&2 "WARNING: Source file $pri_adoc not present in the translation configuration (po4a.conf)"
	done
	# Set up for a newly translatable document if necessary.
	mkdir -p translations/
	[[ -e translations/index.pot ]] || touch translations/index.pot
	# --force is used because Git usage has probably made timestamps unrelated to actual modification time.
	po4a -v po4a.conf --force
	# If index.pot has only one changed line, it's the "POT-Creation-Date", and we don't want to commit that.
	if [[ $(git diff --numstat translations/index.pot | cut -f 1) -eq 1 ]]; then
		echo "No change to index.pot"
		git checkout translations/index.pot
	else
		git add translations/index.pot
	fi
	for lang in translations/??.po translations/??-??.po translations/??-???.po; do
		langcode=$(basename $lang .po)
		# Replace include and image links to translated alternatives
		for tdoc in $(grep -l -E -e '\.[a-z][a-z]\.adoc' -- **/*.$langcode.adoc); do
			echo "Replacing includes in $tdoc for $langcode"
			perl -pi -e 's/([A-Za-z0-9_-]+).'$PRIMARY_LANGUAGE'.adoc/\1.'$langcode'.adoc/' "$tdoc"
		done
		for tdoc in $(grep -l -E -e '\.[a-z][a-z]\.txt' -- **/*.$langcode.adoc); do
			echo "Replacing includes in $tdoc for $langcode"
			perl -pi -e 's/([A-Za-z0-9_-]+).'$PRIMARY_LANGUAGE'.txt/\1.'$langcode'.txt/' "$tdoc"
		done
	done
	echo "Translating source files completed"
else
	echo "No po4a.conf exists, this document will not be translated using PO4A"
fi
echo

# Spell-check the document using Aspell
if [[ -z $no_spellcheck ]]; then
	set +e
	echo -e "${RED}Checking document spelling${NC}"
	echo "(NB many names and technical terms are not in the dictionary.)"

	if [[ -e index.en.adoc ]]; then
		echo "English spellcheck uses Oxford English dictionary."
		for d in *.en.adoc; do
			echo "Checking $d…"
			# Lowercase words
			sed --regexp-extended --file /adoc/spelling-skips.sed < "$d" | \
				aspell --ignore=3 -d en_GB-ize-w_accents -p /adoc/asciidoc.dict list | \
				grep -v '[A-Z]' | sed 's/^/	/'
			# First letter uppercase
			sed --regexp-extended --file /adoc/spelling-skips.sed < "$d" | \
				aspell --ignore=3 -d en_GB-ize-w_accents -p /adoc/asciidoc.dict list | \
				grep '^[A-Z]' | sed 's/^/	/'
			# Subsequent letter uppercase
			sed --regexp-extended --file /adoc/spelling-skips.sed < "$d" | \
				aspell --ignore=3 -d en_GB-ize-w_accents -p /adoc/asciidoc.dict list | \
				grep '^[^A-Z].*[A-Z]' | sed 's/^/	/'
		done
		echo
	fi

	if [[ -e index.es.adoc ]]; then
		echo -e "Spanish spellcheck."
		for d in *.es.adoc; do
			echo "Checking $d…"
			# Lowercase words
			sed --regexp-extended --file /adoc/spelling-skips.sed < "$d" | \
				aspell --ignore=3 -d es -p /adoc/asciidoc-es.dict list | \
				grep -v '[A-Z]' | sed 's/^/	/'
			# First letter uppercase
			sed --regexp-extended --file /adoc/spelling-skips.sed < "$d" | \
				aspell --ignore=3 -d es -p /adoc/asciidoc-es.dict list | \
				grep '^[A-Z]' | sed 's/^/	/'
			# Subsequent letter uppercase
			sed --regexp-extended --file /adoc/spelling-skips.sed < "$d" | \
				aspell --ignore=3 -d es -p /adoc/asciidoc-es.dict list | \
				grep '^[^A-Z].*[A-Z]' | sed 's/^/	/'
		done
		echo
	fi

	set -e
fi

improve_url=https://docs.gbif.org/contributing/
if [[ -z $no_git ]] && git remote get-url origin &> /dev/null; then
	contribute_url=https://github.com/gbif/$(git remote get-url origin | rev | cut -d/ -f1 | rev | cut -d. -f1)
	issue_url=$contribute_url/issues/new
else
	contribute_url=
	issue_url=
fi

# Generate the output HTML and PDF.
rm -f ??/*.html ??/*.html.brotli ??/present/*.html ??/*.pdf ??/*.pdf.brotli *.??.asis \
   ??-??/*.html ??-??/*.html.brotli ??-??/present/*.html ??-??/*.pdf ??-??/*.pdf.brotli *.??-??.asis \
   ??-???/*.html ??-???/*.html.brotli ??-???/present/*.html ??-???/*.pdf ??-???/*.pdf.brotli *.??-???.asis \
   **/lunr-index.json*
for lang in $PRIMARY_LANGUAGE \
			translations/??.po translations/??-??.po translations/??-???.po \
			translations/index.??.adoc translations/index.??-??.adoc translations/index.??-???.adoc; do
	langcode=$(echo $(basename $(basename $lang .po) .adoc) | sed 's/index\.//')
	echo -e "${RED}Building document in language $langcode${NC}"
	mkdir -p $langcode

	if [[ -e index.$langcode.adoc ]]; then
		index=index.$langcode.adoc
	else
		index=translations/index.$langcode.adoc
	fi

	# Document title for PDF filename
	title=$(grep -m 1 '^=[^=]' $index | rev | cut -d: -f2- | rev | unidecode | tr '[:upper:]`~!@"#$%^&*(){}[]/=?+_<>,.\| '"'" '[:lower:]-----------------------------'"-" | sed 's/--*/-/g; s/^-*//; s/-*$//;')
	echo "Document title in $langcode is “$title”"

	# Only add the BibTeX plugin if required
	if grep -q :bibtex-file: $index; then
		echo "Activating BibTeX plugin"
		bibtex_plugin="-r asciidoctor-bibtex"
	else
		bibtex_plugin=
	fi

	# Only add the Multipage plugin if configured
	if grep -q :multipage: $index; then
		echo "Multi-page document"
		multipage_plugin="-b gbif_multipage_html5"
	else
		echo "Single page document"
		multipage_plugin="-b html5"
	fi

	echo -e "${RED}Generating $langcode HTML${NC}"
	asciidoctor \
		-a icons=font \
		-a lang=$langcode \
		-a imagesdir=../ \
		-a sectanchors \
		-a idprefix=@ \
		-a idseparator=-@ \
		-a pdf_filename=$title.$langcode.pdf \
		-a improve_url=$improve_url@ \
		-a contribute_url=$contribute_url@ \
		-a issue_url=$issue_url@ \
		-a stylesheet=/adoc/gbif-stylesheet/stylesheets/gbif.css@ \
		-a plausible-analytics-data-domain=docs.gbif.org,gbif.org \
		$multipage_plugin \
		-r asciidoctor-diagram \
		$bibtex_plugin \
		-r asciidoctor-question \
		-r /adoc/asciidoctor-extensions-lab/lib/git-metadata-preprocessor.rb \
		-r /adoc/gbif-extensions/lib/glossary.rb \
		-r /adoc/gbif-extensions/lib/license-url-docinfoprocessor.rb \
		-r /adoc/gbif-extensions/lib/plausible-analytics-docinfoprocessor.rb \
		-r /adoc/gbif-extensions/lib/term.rb \
		-r /adoc/gbif-extensions/lib/translate-labels.rb \
		-r /adoc/gbif-extensions/lib/translation-links.rb \
		-r /adoc/GbifHtmlConverter.rb \
		-o $langcode/index.$langcode.html \
		--trace \
		--verbose \
		$index

	echo -e "${RED}Generating $langcode search index${NC}"
	node /adoc/lunr/generate-index.js $langcode $langcode/lunr-index.json > lunr.log || \
		(echo "${RED}Lunr index failed"; echo '{}' > $langcode/lunr-index.json)

	if [[ -z $no_compress ]]; then
		brotli --keep --force --output=$langcode/lunr-index.json.brotli --best $langcode/lunr-index.json &
	fi

	echo

	if [[ -n $presentation ]]; then
		echo -e "${RED}Generating $langcode presentation${NC}"
		asciidoctor-revealjs \
			-b revealjs \
			-a icons=font \
			-a lang=$langcode \
			-a imagesdir=../../ \
			-a sectanchors \
			-a idprefix=@ \
			-a idseparator=-@ \
			-a revealjs_theme=sky \
			-a revealjsdir=https://cdn.jsdelivr.net/npm/reveal.js@3.9.2 \
			-a plausible-analytics-data-domain=docs.gbif.org,gbif.org \
			-r asciidoctor-diagram \
			$bibtex_plugin \
			-r asciidoctor-question \
			-r /adoc/asciidoctor-extensions-lab/lib/git-metadata-preprocessor.rb \
			-r /adoc/gbif-extensions/lib/glossary.rb \
			-r /adoc/gbif-extensions/lib/license-url-docinfoprocessor.rb \
			-r /adoc/gbif-extensions/lib/plausible-analytics-docinfoprocessor.rb \
			-r /adoc/gbif-extensions/lib/term.rb \
			-r /adoc/gbif-extensions/lib/translate-labels.rb \
			-r /adoc/gbif-extensions/lib/translation-links.rb \
			-r /adoc/GbifHtmlConverter.rb \
			-o $langcode/present/index.$langcode.html \
			--trace \
			--verbose \
			$index

		echo
	else
		echo -e "${RED}Not generating $langcode presentation${NC}"
	fi

	if [[ -z $no_compress ]]; then
		find $langcode -type f -iname '*.html' -print0 | parallel --will-cite --no-run-if-empty -0 brotli --keep --force --suffix=.brotli --best '{}' &
	fi

	if [[ -n $no_pdf ]]; then
		echo -e "${RED}Not generating $langcode PDF${NC}"
	else
		echo -e "${RED}Generating $langcode PDF${NC}"

		# Use absolute URLs in PDFs.
		if [[ -n $url ]]; then
			sed --in-place "s|link:../|link:$url|" **/*.$langcode.adoc
		fi

		asciidoctor-pdf \
			-a icons=font \
			-a pagenums \
			-a lang=$langcode \
			-a idprefix=@ \
			-a idseparator=-@ \
			-a title-page \
			-a pdf-theme=/adoc/gbif-theme/gbif-theme.yml \
			-a pdf-fontsdir='/adoc/fonts;GEM_FONTS_DIR' \
			-r asciidoctor-diagram \
			$bibtex_plugin \
			-r asciidoctor-question \
			-r /adoc/asciidoctor-extensions-lab/lib/git-metadata-preprocessor.rb \
			-r /adoc/gbif-extensions/lib/glossary.rb \
			-r /adoc/gbif-extensions/lib/term.rb \
			-r /adoc/gbif-extensions/lib/translate-labels.rb \
			-o $langcode/$title.$langcode.pdf \
			--trace \
			--verbose \
			$index

		if [[ -z $no_compress ]]; then
			brotli --keep --force --output=$langcode/$title.$langcode.pdf.brotli --best $langcode/$title.$langcode.pdf &
		fi

		echo
	fi

	if [[ ! -e translations/$langcode.hidden ]]; then
		cat > index.$langcode.asis <<-EOF
			Status: 303 See Other
			Location: ./$langcode/
			Content-Type: text/html; charset=UTF-8
			Content-Language: $langcode
			Vary: negotiate,accept-language

			See <a href="./$langcode/">$langcode</a>.
		EOF
	fi

	echo "$langcode completed"
	echo
done

if [[ -n $no_compress ]]; then
	echo -e "${RED}Not compressing files${NC}"
else
	echo -e "${RED}Compressing files${NC}"

	echo "Compressing SVG files with Brotli compression"
	find . -type f -iname '*.svg' -print0 | parallel --will-cite --no-run-if-empty -0 brotli --keep --force --suffix=.brotli --best '{}'

	echo "Compressing PNG files to WebP format"
	find . -type f -iname '*.png' -size -10000001c -print0 | parallel --will-cite --no-run-if-empty -0 cwebp -short -near_lossless 10 -o '{}.webp' '{}'

	echo "Compressing JPG files to WebP format"
	find . -type f -iname '*.jp*g' -size -10000001c -print0 | parallel --will-cite --no-run-if-empty -0 cwebp -short -o '{}.webp' '{}'
fi
echo

# Add translated subtitles as text streams within the videos
if [[ -n $no_video ]]; then
	echo -e "${RED}Not processing videos${NC}"
elif compgen -G 'videos/*.vtt' > /dev/null; then
	echo -e "${RED}Processing videos${NC}"

	for i in videos/*.mp4; do
		vid=${i/.mp4/}
		if compgen -G "$vid.*.vtt" > /dev/null; then
			echo -n "$vid.mp4: Found these translations: "
			ffmpeg_command=(ffmpeg -i $i -loglevel level+warning)
			for vtt in $vid.*.vtt; do
				ffmpeg_command+=(-i $vtt)
			done
			ffmpeg_command+=(-map 0:v -map 0:a)
			c=1
			for vtt in $vid.*.vtt; do
				ffmpeg_command+=(-map $c)
				c=$((c + 1))
			done
			ffmpeg_command+=(-c:v copy -c:a copy -scodec mov_text)
			c=0
			for vtt in $vid.*.vtt; do
				file_with_lang=$(basename $vtt .vtt)
				lang=${file_with_lang##*.}
				lang3=$(curl -Ss https://api.gbif.org/v1/enumeration/language | jq -r ".[] | select(.iso2 == \"$lang\").iso3")
				echo -n "$lang/$lang3 "
				ffmpeg_command+=(-metadata:s:s:$c language=$lang3)
				c=$((c + 1))
			done
			echo "— inserting into the video."
			ffmpeg_command+=(-n $i.mp4)

			${ffmpeg_command[@]}
			rm -f $i
			mv $i.mp4 $i
		else
			echo "$vid.mp4: No translation VTT files found"
		fi
	done
else
	echo "NO VIDEOS"
	ls videos/*.vtt
fi
echo

if [[ -z $no_compress ]]; then
	echo "Waiting for HTML and PDF compression to complete"
	echo
fi
wait

echo -e "${RED}Document build completed.${NC}"
echo
# Make translation template
# po4a-gettextize -f asciidoc -M utf-8 -m index.adoc -p translations/index.pot

# Update translation
# po4a-updatepo -f asciidoc -M utf-8 -m index.adoc -p translations/da.po

# po4a-normalize -f asciidoc -M utf-8 translations/da.po

# Translate
# po4a-translate -f asciidoc -M utf-8 -m index.adoc -p translations/da.po -k 0 -l index.da.adoc