Skip to content

Production db notclean #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 19 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 9 additions & 101 deletions .github/workflows/qiita-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
matrix:
include:
- cover_package: "qiita_db"
- cover_package: "qiita_pet qiita_core qiita_ware"
# - cover_package: "qiita_pet qiita_core qiita_ware"

services:
postgres:
Expand Down Expand Up @@ -57,6 +57,8 @@ jobs:
# pull out the port so we can modify the configuration file easily
pgport=${{ job.services.postgres.ports[5432] }}
sed -i "s/PORT = 5432/PORT = $pgport/" qiita_core/support_files/config_test.cfg
# SMJ: set to productive mode!
sed -i "s/TEST_ENVIRONMENT = TRUE/TEST_ENVIRONMENT = FALSE/" qiita_core/support_files/config_test.cfg

# PGPASSWORD is read by pg_restore, which is called by the build_db process.
export PGPASSWORD=postgres
Expand Down Expand Up @@ -101,25 +103,7 @@ jobs:
pwd
mkdir ~/.qiita_plugins

- name: Install plugins
shell: bash -l {0}
run: |
wget https://data.qiime2.org/distro/core/qiime2-2022.11-py38-linux-conda.yml
conda env create --quiet -n qtp-biom --file qiime2-2022.11-py38-linux-conda.yml
rm qiime2-2022.11-py38-linux-conda.yml
export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt
export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
export REDBIOM_HOST="http://localhost:7379"
conda activate qtp-biom
pip install -U pip
pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
pip install https://github.com/qiita-spots/qtp-biom/archive/master.zip

# if QIITA_ROOTCA_CERT is appended to certifi's cacert.pem file,
# then --server-cert does not need to be specified to any plugin,
# including configure_biom.
# echo `python -c "import certifi;print(certifi.where())"`
configure_biom --env-script "source /home/runner/.profile; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT
# SMJ: don't install any plugins! ~/.qiita_plugins directory should remain empty

- name: Starting services
shell: bash -l {0}
Expand Down Expand Up @@ -156,84 +140,8 @@ jobs:
conda activate qiita
# adapt environment_script for private qiita plugins from travis to github actions.
sed 's#export PATH="/home/travis/miniconda3/bin:$PATH"; source #source /home/runner/.profile; conda #' -i qiita_db/support_files/patches/54.sql
qiita-env make --no-load-ontologies
qiita-test-install
qiita plugins update

echo "6. Starting supervisord => multiple qiita instances"
supervisord -c ${PWD}/qiita_pet/supervisor_example.conf
sleep 10
cat /tmp/supervisord.log

echo "7. Starting plugins"
conda deactivate
conda activate qtp-biom
export QIITA_CLIENT_DEBUG_LEVEL=DEBUG
start_biom https://localhost:8383 register ignored
conda deactivate

echo "8. Setting up SSH"
ssh-keygen -t rsa -b 4096 -N '' -f $PWD/qiita_ware/test/test_data/test_key
mkdir ~/.ssh/
cp $PWD/qiita_ware/test/test_data/test_key* ~/.ssh/
cat ~/.ssh/test_key.pub > ~/.ssh/authorized_keys
chmod 600 $PWD/qiita_ware/test/test_data/test_key*
chmod 600 ~/.ssh/*
chmod 700 ~/.ssh/
echo "Connecting as $USER@localhost"
# this line (and the -o StrictHostKeyChecking=no) is so the server
# is added to the list of known servers
scp -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1

- name: Main tests
shell: bash -l {0}
env:
COVER_PACKAGE: ${{ matrix.cover_package }}
run: |
conda activate qiita
export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt
export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
# for testing we only need to have this set, not actually exist
export QIITA_JOB_SCHEDULER_EPILOGUE=`/path/to/epilogue/file`
export REDBIOM_HOST="http://localhost:7379"

nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=${COVER_PACKAGE// / --cover-package=} -e 'test_submit_EBI_parse_EBI_reply_failure' -e 'test_full_submission'

# killing the qiita server to run the next commands
QIITA_PID=`cat /tmp/supervisord.pid`
kill $QIITA_PID
sleep 10
if [[ "$COVER_PACKAGE" != *"qiita_db"* ]]; then test_data_studies/commands.sh; all-qiita-cron-job; fi

- name: Submit coveralls
uses: AndreMiras/coveralls-python-action@develop
with:
github-token: ${{ secrets.github_token }}
flag-name: "${{ matrix.cover_package }}"
parallel: true

coveralls_finish:
needs: main
runs-on: ubuntu-latest
steps:
- name: Coveralls Finished
uses: AndreMiras/coveralls-python-action@develop
with:
github-token: ${{ secrets.github_token }}
parallel-finished: true

lint:
runs-on: ubuntu-latest
steps:
- name: flake8
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: install dependencies
run: python -m pip install --upgrade pip
- name: Check out repository code
uses: actions/checkout@v2
- name: lint
run: |
pip install -q flake8
flake8 qiita_* setup.py scripts/qiita* notebooks/*/*.py
qiita-env make #--no-load-ontologies

PGPASSWORD=postgres psql -U postgres --host=localhost --port=32768 -d qiita_test -c "SELECT * FROM qiita.software;"
PGPASSWORD=postgres psql -U postgres --host=localhost --port=32768 -d qiita_test -c "SELECT * FROM qiita.default_workflow;"

3 changes: 3 additions & 0 deletions qiita_db/sql_connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,15 @@ def _open_connection(self):
host=qiita_config.host,
port=qiita_config.port)
self._connection.autocommit = True
print("STEFAN:admin", qiita_config.admin_user, qiita_config.admin_password, qiita_config.host, qiita_config.port)
else:
self._connection = connect(user=qiita_config.user,
password=qiita_config.password,
database=qiita_config.database,
host=qiita_config.host,
port=qiita_config.port)
print("STEFAN:user", qiita_config.user, qiita_config.password, qiita_config.database, qiita_config.host, qiita_config.port)

except OperationalError as e:
# catch three known common exceptions and raise runtime errors
try:
Expand Down