Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 31 additions & 38 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,47 +1,40 @@
FROM postgres:11

ARG DEBIAN_FRONTEND=noninteractive
ARG BUILD_DEPS='gcc git libffi-dev libssl-dev python3-dev python3-pip python3-wheel'

RUN apt-get update && \
apt-get install -y --no-install-recommends \
$BUILD_DEPS \
gosu \
lzop \
libpq-dev \
pv \
python3 \
util-linux \
# swift package needs pkg_resources and setuptools
python3-pkg-resources \
python3-setuptools \
python3-pip && \
ln -sf /usr/bin/python3 /usr/bin/python && \
ln -sf /usr/bin/pip3 /usr/bin/pip

# setuptools from ubuntu archives is too old for googleapis-common-protos
RUN pip install --upgrade setuptools && \
pip install --disable-pip-version-check --no-cache-dir \
envdir==1.0.1 \
wal-e[aws,azure,google,swift]==1.1.0 \
gcloud==0.18.3 \
oauth2client==4.1.3 \
azure-storage==0.20.0

# cleanup
RUN apt-get purge -y --auto-remove $BUILD_DEPS && \
apt-get autoremove -y && \
apt-get clean -y
FROM postgres:11-alpine

ENV WALE_LOG_DESTINATION stderr
ENV WALE_ENVDIR /etc/wal-e.d/env

RUN mkdir -p $WALE_ENVDIR \
&& echo 'http://dl-cdn.alpinelinux.org/alpine/v3.5/main' >> /etc/apk/repositories \
&& apk add --update --virtual .build-deps \
git \
build-base \
libffi-dev \
openssl-dev \
python3-dev=3.5.6-r0 \
linux-headers \
&& apk add \
lzo \
pv \
util-linux \
ca-certificates \
python3=3.5.6-r0 \
&& pip3 install --upgrade pip setuptools \
&& pip install --disable-pip-version-check --no-cache-dir \
psycopg2-binary==2.7.6.1 \
envdir==1.0.1 \
wal-e[aws,azure,google,swift]==1.1.0 \
gcloud==0.18.3 \
oauth2client==4.1.3 \
azure-storage==0.20.0 \
&& apk del .build-deps \
&& rm -rf /var/cache/apk/*

COPY rootfs /
ENV WALE_ENVDIR=/etc/wal-e.d/env
RUN mkdir -p $WALE_ENVDIR

ARG PATCH_CMD="python3 /patcher-script.py"
RUN $PATCH_CMD file /bin/create_bucket /patcher-script.d/patch_boto_s3.py
RUN $PATCH_CMD file /usr/local/bin/wal-e /patcher-script.d/patch_boto_s3.py
RUN $PATCH_CMD module wal_e.cmd /patcher-script.d/patch_boto_s3.py
RUN $PATCH_CMD module wal_e.worker.worker_util /patcher-script.d/patch_wal_e_s3.py


CMD ["/docker-entrypoint.sh", "postgres"]
EXPOSE 5432
6 changes: 5 additions & 1 deletion charts/database/templates/database-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,15 @@ spec:
value: "{{.Values.global.storage}}"
- name: PGCTLTIMEOUT
value: "{{.Values.postgres.timeout}}"
{{- if eq .Values.global.storage "s3" }}
- name: S3_SSE
value: "{{.Values.s3.use_sse}}"
{{- end}}
lifecycle:
preStop:
exec:
command:
- gosu
- su-exec
- postgres
- do_backup
readinessProbe:
Expand Down
2 changes: 1 addition & 1 deletion rootfs/bin/create_bucket
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3

import os

Expand Down
1 change: 1 addition & 0 deletions rootfs/bin/do_backup
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ export BACKUPS_TO_RETAIN=${BACKUPS_TO_RETAIN:-5}
echo "Performing a base backup..."
if [[ -f "$PGDATA/recovery.conf" ]] ; then
echo "Database is currently recovering from a backup. Aborting"
sleep 9
else
# perform a backup
envdir "$WALE_ENVDIR" wal-e backup-push "$PGDATA"
Expand Down
2 changes: 1 addition & 1 deletion rootfs/bin/is_running
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ if [[ -f "$PGDATA/recovery.conf" ]]; then
exit 1
fi

gosu postgres pg_ctl status
su-exec postgres pg_ctl status
5 changes: 5 additions & 0 deletions rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ if [[ "$DATABASE_STORAGE" == "s3" || "$DATABASE_STORAGE" == "minio" ]]; then
else
echo "https+path://s3-${AWS_REGION}.amazonaws.com:443" > WALE_S3_ENDPOINT
fi
if [[ $S3_SSE ]]; then
echo $S3_SSE > WALE_S3_SSE
else
echo "false" > WALE_S3_SSE
fi
else
AWS_REGION="us-east-1"
BUCKET_NAME="dbwal"
Expand Down
15 changes: 3 additions & 12 deletions rootfs/docker-entrypoint-initdb.d/003_restore_from_backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ chmod 0700 "$PGDATA"

# reboot the server for wal_level to be set before backing up
echo "Rebooting postgres to enable archive mode"
gosu postgres pg_ctl -D "$PGDATA" -w restart
su-exec postgres pg_ctl -D "$PGDATA" -w restart

# check if there are any backups -- if so, let's restore
# we could probably do better than just testing number of lines -- one line is just a heading, meaning no backups
if [[ $(envdir "$WALE_ENVDIR" wal-e --terse backup-list | wc -l) -gt "1" ]]; then
echo "Found backups. Restoring from backup..."
gosu postgres pg_ctl -D "$PGDATA" -w stop
su-exec postgres pg_ctl -D "$PGDATA" -w stop
rm -rf "$PGDATA/*"
envdir "$WALE_ENVDIR" wal-e backup-fetch "$PGDATA" LATEST
cat << EOF > "$PGDATA/postgresql.conf"
Expand Down Expand Up @@ -50,20 +50,11 @@ EOF
echo "restore_command = 'envdir /etc/wal-e.d/env wal-e wal-fetch \"%f\" \"%p\"'" >> "$PGDATA/recovery.conf"
chown -R postgres:postgres "$PGDATA"
chmod 0700 "$PGDATA"
gosu postgres pg_ctl -D "$PGDATA" \
su-exec postgres pg_ctl -D "$PGDATA" \
-o "-c listen_addresses=''" \
-w start

echo "Waiting for recovery completion..."
while [ ! -f "$PGDATA/recovery.done" ]
do
sleep 2
done
fi

echo "Performing an initial backup..."
gosu postgres envdir "$WALE_ENVDIR" wal-e backup-push "$PGDATA"

# ensure $PGDATA has the right permissions
chown -R postgres:postgres "$PGDATA"
chmod 0700 "$PGDATA"
2 changes: 1 addition & 1 deletion rootfs/docker-entrypoint-initdb.d/004_run_backups.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env bash

# Run periodic backups in the background
gosu postgres backup &
su-exec postgres backup &
8 changes: 4 additions & 4 deletions rootfs/docker-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ if [ "$1" = 'postgres' ]; then

# look specifically for PG_VERSION, as it is expected in the DB dir
if [ ! -s "$PGDATA/PG_VERSION" ]; then
gosu postgres initdb
su-exec postgres initdb

# check password first so we can output the warning before postgres
# messes it up
Expand Down Expand Up @@ -54,7 +54,7 @@ if [ "$1" = 'postgres' ]; then

# internal start of server in order to allow set-up using psql-client
# does not listen on TCP/IP and waits until start finishes
gosu postgres pg_ctl -D "$PGDATA" \
su-exec postgres pg_ctl -D "$PGDATA" \
-o "-c listen_addresses=''" \
-w start

Expand Down Expand Up @@ -94,15 +94,15 @@ if [ "$1" = 'postgres' ]; then
echo
done

gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop
su-exec postgres pg_ctl -D "$PGDATA" -m fast -w stop
set_listen_addresses '*'

echo
echo 'PostgreSQL init process complete; ready for start up.'
echo
fi

exec gosu postgres "$@"
exec su-exec postgres "$@"
fi

exec "$@"
6 changes: 3 additions & 3 deletions rootfs/patcher-script.d/patch_wal_e_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ def wrap_uri_put_file(creds, uri, fp, content_type=None, conn=None):
k = s3_util._uri_to_key(creds, uri, conn=conn)
if content_type is not None:
k.content_type = content_type
encrypt_key = False
if os.getenv('DATABASE_STORAGE') == 's3':
encrypt_key=True
else:
encrypt_key=False
if os.getenv('WALE_S3_SSE', 'false') == 'true':
encrypt_key = True
k.set_contents_from_file(fp, encrypt_key=encrypt_key)
return k
s3.uri_put_file = wrap_uri_put_file
Expand Down