[PATCH v3 1/6] settings: unify database connection variables
You-Sheng Yang
vicamo at gmail.com
Thu Dec 9 04:02:33 AEDT 2021
This changeset replaces all PW_TEST_DB_* with corresponding DATABASE_*
variables, fixes inconsistent DATABASE_PASS/DATABASE_PASSWORD usages,
and moves database settings into base.py for sharing/syncing between dev
and production setup.
Signed-off-by: You-Sheng Yang <vicamo at gmail.com>
---
.github/workflows/ci.yaml | 40 +++++++++++--------
docker-compose-pg.yml | 12 +++---
docker-compose.yml | 10 +++--
docs/deployment/installation.rst | 4 +-
docs/development/installation.rst | 13 +++---
lib/uwsgi/patchwork.ini | 2 +-
patchwork/settings/base.py | 39 ++++++++++++++++++
patchwork/settings/dev.py | 28 +------------
patchwork/settings/production.example.py | 17 --------
tools/docker/entrypoint.sh | 51 ++++++++++++++++--------
tox.ini | 4 +-
11 files changed, 123 insertions(+), 97 deletions(-)
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f4a33b3..d37cdbf 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -25,13 +25,20 @@ jobs:
matrix:
python: [3.6, 3.7, 3.8, 3.9]
db: [postgres, mysql]
+ env:
+ DATABASE_TYPE: "${{ matrix.db }}"
+ DATABASE_HOST: "127.0.0.1"
+ DATABASE_NAME: patchwork
+ DATABASE_USER: patchwork
+ DATABASE_PASSWORD: password
+ MYSQL_ROOT_PASSWORD: root
services:
postgres:
image: postgres:latest
env:
- POSTGRES_DB: patchwork
- POSTGRES_PASSWORD: patchwork
- POSTGRES_USER: patchwork
+ POSTGRES_DB: ${{ env.DATABASE_NAME }}
+ POSTGRES_PASSWORD: ${{ env.DATABASE_PASSWORD }}
+ POSTGRES_USER: ${{ env.DATABASE_USER }}
ports:
- 5432:5432
options: >-
@@ -42,10 +49,10 @@ jobs:
mysql:
image: mysql:latest
env:
- MYSQL_DATABASE: patchwork
- MYSQL_USER: patchwork
- MYSQL_PASSWORD: patchwork
- MYSQL_ROOT_PASSWORD: root
+ MYSQL_DATABASE: ${{ env.DATABASE_NAME }}
+ MYSQL_USER: ${{ env.DATABASE_USER }}
+ MYSQL_PASSWORD: ${{ env.DATABASE_PASSWORD }}
+ MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }}
ports:
- 3306:3306
options: >-
@@ -64,22 +71,23 @@ jobs:
run: python -m pip install tox tox-gh-actions codecov
- name: Log database configuration (mysql)
if: ${{ matrix.db == 'mysql' }}
- run: mysql -h 127.0.0.1 -e "SELECT VERSION(), CURRENT_USER();" -uroot -proot patchwork
+ run: |
+ mysql -h 127.0.0.1 -e "SELECT VERSION(), CURRENT_USER();" \
+ -uroot -p${MYSQL_ROOT_PASSWORD} ${DATABASE_NAME}
- name: Log database configuration (postgres)
if: ${{ matrix.db == 'postgres' }}
- run: psql -h 127.0.0.1 -c "SELECT VERSION(), CURRENT_USER, current_database()" -U patchwork -d patchwork
+ run: |
+ psql -h 127.0.0.1 -c "SELECT VERSION(), CURRENT_USER, current_database()" \
+ -U ${DATABASE_USER} -d ${DATABASE_NAME}
env:
- PGPASSWORD: patchwork
+ PGPASSWORD: ${{ env.DATABASE_PASSWORD }}
- name: Modify database user permissions (mysql)
if: ${{ matrix.db == 'mysql' }}
- run: mysql -h 127.0.0.1 -e "GRANT ALL ON \`test\\_patchwork%\`.* to 'patchwork'@'%';" -uroot -proot
+ run: |
+ mysql -h 127.0.0.1 -e "GRANT ALL ON \`test\\_${DATABASE_NAME}%\`.* to '${DATABASE_USER}'@'%';" \
+ -uroot -p${MYSQL_ROOT_PASSWORD}
- name: Run unit tests (via tox)
run: tox
- env:
- PW_TEST_DB_TYPE: "${{ matrix.db }}"
- PW_TEST_DB_USER: "patchwork"
- PW_TEST_DB_PASS: "patchwork"
- PW_TEST_DB_HOST: "127.0.0.1"
docs:
name: Build docs
runs-on: ubuntu-latest
diff --git a/docker-compose-pg.yml b/docker-compose-pg.yml
index 44c49fb..7398929 100644
--- a/docker-compose-pg.yml
+++ b/docker-compose-pg.yml
@@ -24,9 +24,9 @@ services:
environment:
- UID
- GID
- - PW_TEST_DB_HOST=db
- - PW_TEST_DB_PORT=5432
- - PW_TEST_DB_TYPE=postgres
- - PW_TEST_DB_USER=postgres
- - PW_TEST_DB_PASS=password
- - PGPASSWORD=password
+ - DATABASE_TYPE=postgres
+ - DATABASE_HOST=db
+ - DATABASE_PORT=5432
+ - DATABASE_NAME=patchwork
+ - DATABASE_USER=postgres
+ - DATABASE_PASSWORD=password
diff --git a/docker-compose.yml b/docker-compose.yml
index 1d49c51..103f19f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
volumes:
- ./tools/docker/db/data:/var/lib/mysql
environment:
- - MYSQL_ROOT_PASSWORD=password
+ - MYSQL_ROOT_PASSWORD=root
- MYSQL_USER=patchwork
- MYSQL_PASSWORD=password
@@ -26,5 +26,9 @@ services:
environment:
- UID
- GID
- - PW_TEST_DB_HOST=db
- - PW_TEST_DB_PORT=3306
+ # skip DATABASE_TYPE explicitly as mysql should be the default type.
+ - DATABASE_HOST=db
+ - DATABASE_PORT=3306
+ - DATABASE_USER=patchwork
+ - DATABASE_PASSWORD=password
+ - MYSQL_ROOT_PASSWORD=root
diff --git a/docs/deployment/installation.rst b/docs/deployment/installation.rst
index 1abf187..14ad746 100644
--- a/docs/deployment/installation.rst
+++ b/docs/deployment/installation.rst
@@ -64,7 +64,7 @@ used to ease deployment:
Username that the Patchwork web application will access the database with. We
will use ``www-data``, for reasons described later in this guide.
-``DATABASE_PASS=``
+``DATABASE_PASSWORD=``
Password that the Patchwork web application will access the database with. As
we're going to use *peer* authentication (more on this later), this will be
unset.
@@ -323,7 +323,7 @@ Once done, we should be able to check that all requirements are met using the
$ sudo -u www-data \
--preserve-env=DATABASE_NAME \
--preserve-env=DATABASE_USER \
- --preserve-env=DATABASE_PASS \
+ --preserve-env=DATABASE_PASSWORD \
--preserve-env=DATABASE_HOST \
--preserve-env=DATABASE_PORT \
--preserve-env=STATIC_ROOT \
diff --git a/docs/development/installation.rst b/docs/development/installation.rst
index d16177c..3507f43 100644
--- a/docs/development/installation.rst
+++ b/docs/development/installation.rst
@@ -316,7 +316,7 @@ if necessary:
The ``patchwork`` username and ``password`` password are the defaults
expected by the provided ``dev`` settings files. If using something
- different, export the ``PW_TEST_DB_USER`` and ``PW_TEST_DB_PASS`` variables
+ different, export the ``DATABASE_USER`` and ``DATABASE_PASSWORD`` variables
described in the :ref:`Environment Variables <dev-envvar>` section below.
Alternatively, you can create your own settings file with these variables
hardcoded and change the value of ``DJANGO_SETTINGS_MODULE`` as described
@@ -470,17 +470,16 @@ __ https://django-dbbackup.readthedocs.io/en/stable/
Environment Variables
---------------------
-The following environment variables are available to configure settings when
-using the provided ``dev`` settings file.
+The following environment variables are available to configure settings.
-``PW_TEST_DB_NAME=patchwork``
+``DATABASE_NAME=patchwork``
Name of the database
-``PW_TEST_DB_USER=patchwork``
+``DATABASE_USER=patchwork``
Username to access the database with
-``PW_TEST_DB_PASS=password``
+``DATABASE_PASSWORD=password``
Password to access the database with<
-``PW_TEST_DB_TYPE=mysql``
+``DATABASE_TYPE=mysql``
Type of database to use. Options: ``mysql``, ``postgres``
diff --git a/lib/uwsgi/patchwork.ini b/lib/uwsgi/patchwork.ini
index 95a0613..dbff508 100644
--- a/lib/uwsgi/patchwork.ini
+++ b/lib/uwsgi/patchwork.ini
@@ -10,7 +10,7 @@ url = /
# env = DJANGO_SECRET_KEY=
# env = DATABASE_NAME=
# env = DATABASE_USER=
-# env = DATABASE_PASS=
+# env = DATABASE_PASSWORD=
# env = DATABASE_HOST=
# env = DATABASE_PORT=
# env = STATIC_ROOT=
diff --git a/patchwork/settings/base.py b/patchwork/settings/base.py
index ff14d91..5e200f0 100644
--- a/patchwork/settings/base.py
+++ b/patchwork/settings/base.py
@@ -105,6 +105,45 @@ STATICFILES_DIRS = [
os.path.join(ROOT_DIR, 'htdocs'),
]
+# Database
+#
+# If you're using a postgres database, connecting over a local unix-domain
+# socket, then the following setting should work for you. Otherwise,
+# see https://docs.djangoproject.com/en/2.2/ref/settings/#databases
+
+if os.getenv('DATABASE_TYPE', None) == 'postgres':
+ DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.postgresql_psycopg2',
+ 'HOST': os.environ.get('DATABASE_HOST', 'localhost'),
+ 'PORT': os.environ.get('DATABASE_PORT', ''),
+ 'NAME': os.environ.get('DATABASE_NAME', 'patchwork'),
+ 'USER': os.environ.get('DATABASE_USER', 'patchwork'),
+ 'PASSWORD': os.environ.get('DATABASE_PASSWORD', 'password'),
+ },
+ }
+elif os.getenv('DATABASE_TYPE', None) == 'sqlite3':
+ DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.sqlite3',
+ 'NAME': os.environ.get('DATABASE_NAME', ''),
+ },
+ }
+else:
+ DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.mysql',
+ 'HOST': os.getenv('DATABASE_HOST', 'localhost'),
+ 'PORT': os.getenv('DATABASE_PORT', ''),
+ 'NAME': os.getenv('DATABASE_NAME', 'patchwork'),
+ 'USER': os.getenv('DATABASE_USER', 'patchwork'),
+ 'PASSWORD': os.getenv('DATABASE_PASSWORD', 'password'),
+ 'TEST': {
+ 'CHARSET': 'utf8',
+ },
+ },
+ }
+
#
# Third-party application settings
#
diff --git a/patchwork/settings/dev.py b/patchwork/settings/dev.py
index 2b7fc95..cb4cb19 100644
--- a/patchwork/settings/dev.py
+++ b/patchwork/settings/dev.py
@@ -7,8 +7,6 @@ Design based on:
http://www.revsys.com/blog/2014/nov/21/recommended-django-project-layout/
"""
-import os
-
from .base import * # noqa
try:
@@ -36,30 +34,8 @@ SECRET_KEY = '00000000000000000000000000000000000000000000000000' # noqa
DEBUG = True
-DATABASES = {
- 'default': {
- 'ENGINE': 'django.db.backends.mysql',
- 'HOST': os.getenv('PW_TEST_DB_HOST', 'localhost'),
- 'PORT': os.getenv('PW_TEST_DB_PORT', ''),
- 'USER': os.getenv('PW_TEST_DB_USER', 'patchwork'),
- 'PASSWORD': os.getenv('PW_TEST_DB_PASS', 'password'),
- 'NAME': os.getenv('PW_TEST_DB_NAME', 'patchwork'),
- 'TEST': {
- 'CHARSET': 'utf8',
- },
- },
-}
-
-if os.getenv('PW_TEST_DB_TYPE', None) == 'postgres':
- DATABASES['default']['ENGINE'] = 'django.db.backends.postgresql_psycopg2'
- DATABASES['default']['HOST'] = os.getenv('PW_TEST_DB_HOST', '')
-elif os.getenv('PW_TEST_DB_TYPE', None) == 'sqlite':
- DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
- DATABASES['default']['NAME'] = '/dev/shm/patchwork.test.db.sqlite3'
- del DATABASES['default']['HOST']
- del DATABASES['default']['PORT']
- del DATABASES['default']['USER']
- del DATABASES['default']['PASSWORD']
+if DATABASES['default']['ENGINE'] == 'mysql': # noqa: F405
+ DATABASES['default']['TEST'] = {'CHARSET': 'utf8'} # noqa: F405
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
diff --git a/patchwork/settings/production.example.py b/patchwork/settings/production.example.py
index caaf429..ff05629 100644
--- a/patchwork/settings/production.example.py
+++ b/patchwork/settings/production.example.py
@@ -48,23 +48,6 @@ ADMINS = (
# ('Jeremy Kerr', 'jk at ozlabs.org'),
)
-# Database
-#
-# If you're using a postgres database, connecting over a local unix-domain
-# socket, then the following setting should work for you. Otherwise,
-# see https://docs.djangoproject.com/en/2.2/ref/settings/#databases
-
-DATABASES = {
- 'default': {
- 'ENGINE': 'django.db.backends.postgresql_psycopg2',
- 'NAME': os.environ.get('DATABASE_NAME', ''),
- 'USER': os.environ.get('DATABASE_USER', ''),
- 'PASSWORD': os.environ.get('DATABASE_PASSWORD', ''),
- 'HOST': os.environ.get('DATABASE_HOST', ''),
- 'PORT': os.environ.get('DATABASE_PORT', ''),
- },
-}
-
#
# Static files settings
# https://docs.djangoproject.com/en/2.2/ref/settings/#static-files
diff --git a/tools/docker/entrypoint.sh b/tools/docker/entrypoint.sh
index 5450a53..faa36b3 100755
--- a/tools/docker/entrypoint.sh
+++ b/tools/docker/entrypoint.sh
@@ -1,45 +1,62 @@
#!/bin/bash
set -euo pipefail
-PW_TEST_DB_TYPE=${PW_TEST_DB_TYPE:-mysql}
+export DATABASE_HOST=${DATABASE_HOST:-}
+export DATABASE_PORT=${DATABASE_PORT:-}
+export DATABASE_NAME=${DATABASE_NAME:-patchwork}
+export DATABASE_USER=${DATABASE_USER:-patchwork}
+export DATABASE_PASSWORD=${DATABASE_PASSWORD:-password}
+
+case "${DATABASE_TYPE:-}" in
+postgres)
+ export PGPORT=${DATABASE_PORT}
+ export PGPASSWORD=${DATABASE_PASSWORD}
+ psql_args=( ${DATABASE_HOST:+--host=${DATABASE_HOST}} "--username=${DATABASE_USER}" )
+ ;;
+*)
+ export DATABASE_TYPE=mysql
+ mysql_args=( ${DATABASE_HOST:+--host=${DATABASE_HOST}} ${DATABASE_PORT:+--port=${DATABASE_PORT}} "--user=${DATABASE_USER}" "--password=${DATABASE_PASSWORD}" )
+ mysql_root_args=( ${DATABASE_HOST:+--host=${DATABASE_HOST}} ${DATABASE_PORT:+--port=${DATABASE_PORT}} "--user=root" "--password=${MYSQL_ROOT_PASSWORD:-}" )
+ ;;
+esac
# functions
test_db_connection() {
- if [ ${PW_TEST_DB_TYPE} = "postgres" ]; then
- echo ';' | psql -h $PW_TEST_DB_HOST -U postgres 2> /dev/null > /dev/null
+ if [ ${DATABASE_TYPE} = "postgres" ]; then
+ echo ';' | psql "${psql_args[@]}" 2> /dev/null > /dev/null
else
- mysqladmin -h $PW_TEST_DB_HOST -u patchwork --password=password ping > /dev/null 2> /dev/null
+ mysqladmin "${mysql_root_args[@]}" ping > /dev/null 2> /dev/null
fi
}
test_database() {
- if [ ${PW_TEST_DB_TYPE} = "postgres" ]; then
- echo ';' | psql -h $PW_TEST_DB_HOST -U postgres patchwork 2> /dev/null
+ if [ ${DATABASE_TYPE} = "postgres" ]; then
+ echo ';' | psql "${psql_args[@]}" "${DATABASE_NAME}" 2> /dev/null
else
- echo ';' | mysql -h $PW_TEST_DB_HOST -u patchwork -ppassword patchwork 2> /dev/null
+ echo ';' | mysql "${mysql_args[@]}" "${DATABASE_NAME}" 2> /dev/null
fi
}
reset_data_mysql() {
- mysql -uroot -ppassword -h $PW_TEST_DB_HOST << EOF
-DROP DATABASE IF EXISTS patchwork;
-CREATE DATABASE patchwork CHARACTER SET utf8;
-GRANT ALL ON patchwork.* TO 'patchwork' IDENTIFIED BY 'password';
-GRANT ALL ON \`test\\_patchwork%\`.* to 'patchwork'@'%';
+ mysql "${mysql_root_args[@]}" << EOF
+DROP DATABASE IF EXISTS ${DATABASE_NAME};
+CREATE DATABASE ${DATABASE_NAME} CHARACTER SET utf8;
+GRANT ALL ON ${DATABASE_NAME}.* TO '${DATABASE_USER}' IDENTIFIED BY '${DATABASE_PASSWORD}';
+GRANT ALL ON \`test\\_${DATABASE_NAME}%\`.* to '${DATABASE_USER}'@'%';
FLUSH PRIVILEGES;
EOF
}
reset_data_postgres() {
- psql -h $PW_TEST_DB_HOST -U postgres <<EOF
-DROP DATABASE IF EXISTS patchwork;
-CREATE DATABASE patchwork WITH ENCODING = 'UTF8';
+ psql "${psql_args[@]}" <<EOF
+DROP DATABASE IF EXISTS ${DATABASE_NAME};
+CREATE DATABASE ${DATABASE_NAME} WITH ENCODING = 'UTF8';
EOF
}
reset_data() {
- if [ x${PW_TEST_DB_TYPE} = x"postgres" ]; then
+ if [ x${DATABASE_TYPE} = x"postgres" ]; then
reset_data_postgres
else
reset_data_mysql
@@ -87,7 +104,7 @@ set -e
# check if db is connected
if ! test_db_connection; then
- echo "The database seems not to be connected, or the patchwork user is broken"
+ echo "The database seems not to be connected, or the ${DATABASE_USER} user is broken"
echo "MySQL/Postgres may still be starting. Waiting 5 seconds."
sleep 5
if ! test_db_connection; then
diff --git a/tox.ini b/tox.ini
index 3275600..374cbb9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,8 +25,8 @@ setenv =
py{37,38,39}: PYTHONWARNINGS = once
passenv =
http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
- PW_TEST_DB_TYPE PW_TEST_DB_USER PW_TEST_DB_PASS PW_TEST_DB_HOST
- PW_TEST_DB_PORT PW_TEST_DB_NAME DJANGO_TEST_PROCESSES
+ DATABASE_TYPE DATABASE_USER DATABASE_PASSWORD DATABASE_HOST
+ DATABASE_PORT DATABASE_NAME DJANGO_TEST_PROCESSES
commands =
python {toxinidir}/manage.py test --noinput --parallel -- {posargs:patchwork}
--
2.32.0
More information about the Patchwork
mailing list