[PATCH v3 1/6] settings: unify database connection variables

You-Sheng Yang vicamo at gmail.com
Thu Dec 9 04:02:33 AEDT 2021


This changeset replaces all PW_TEST_DB_* with corresponding DATABASE_*
variables, fixes inconsistent DATABASE_PASS/DATABASE_PASSWORD usages,
and moves database settings into base.py for sharing/syncing between dev
and production setup.

Signed-off-by: You-Sheng Yang <vicamo at gmail.com>
---
 .github/workflows/ci.yaml                | 40 +++++++++++--------
 docker-compose-pg.yml                    | 12 +++---
 docker-compose.yml                       | 10 +++--
 docs/deployment/installation.rst         |  4 +-
 docs/development/installation.rst        | 13 +++---
 lib/uwsgi/patchwork.ini                  |  2 +-
 patchwork/settings/base.py               | 39 ++++++++++++++++++
 patchwork/settings/dev.py                | 28 +------------
 patchwork/settings/production.example.py | 17 --------
 tools/docker/entrypoint.sh               | 51 ++++++++++++++++--------
 tox.ini                                  |  4 +-
 11 files changed, 123 insertions(+), 97 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f4a33b3..d37cdbf 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -25,13 +25,20 @@ jobs:
       matrix:
         python: [3.6, 3.7, 3.8, 3.9]
         db: [postgres, mysql]
+    env:
+      DATABASE_TYPE: "${{ matrix.db }}"
+      DATABASE_HOST: "127.0.0.1"
+      DATABASE_NAME: patchwork
+      DATABASE_USER: patchwork
+      DATABASE_PASSWORD: password
+      MYSQL_ROOT_PASSWORD: root
     services:
       postgres:
         image: postgres:latest
         env:
-          POSTGRES_DB: patchwork
-          POSTGRES_PASSWORD: patchwork
-          POSTGRES_USER: patchwork
+          POSTGRES_DB: ${{ env.DATABASE_NAME }}
+          POSTGRES_PASSWORD: ${{ env.DATABASE_PASSWORD }}
+          POSTGRES_USER: ${{ env.DATABASE_USER }}
         ports:
           - 5432:5432
         options: >-
@@ -42,10 +49,10 @@ jobs:
       mysql:
         image: mysql:latest
         env:
-          MYSQL_DATABASE: patchwork
-          MYSQL_USER: patchwork
-          MYSQL_PASSWORD: patchwork
-          MYSQL_ROOT_PASSWORD: root
+          MYSQL_DATABASE: ${{ env.DATABASE_NAME }}
+          MYSQL_USER: ${{ env.DATABASE_USER }}
+          MYSQL_PASSWORD: ${{ env.DATABASE_PASSWORD }}
+          MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }}
         ports:
           - 3306:3306
         options: >-
@@ -64,22 +71,23 @@ jobs:
         run: python -m pip install tox tox-gh-actions codecov
       - name: Log database configuration (mysql)
         if: ${{ matrix.db == 'mysql' }}
-        run: mysql -h 127.0.0.1 -e "SELECT VERSION(), CURRENT_USER();" -uroot -proot patchwork
+        run: |
+          mysql -h 127.0.0.1 -e "SELECT VERSION(), CURRENT_USER();" \
+              -uroot -p${MYSQL_ROOT_PASSWORD} ${DATABASE_NAME}
       - name: Log database configuration (postgres)
         if: ${{ matrix.db == 'postgres' }}
-        run: psql -h 127.0.0.1 -c "SELECT VERSION(), CURRENT_USER, current_database()" -U patchwork -d patchwork
+        run: |
+          psql -h 127.0.0.1 -c "SELECT VERSION(), CURRENT_USER, current_database()" \
+              -U ${DATABASE_USER} -d ${DATABASE_NAME}
         env:
-          PGPASSWORD: patchwork
+          PGPASSWORD: ${{ env.DATABASE_PASSWORD }}
       - name: Modify database user permissions (mysql)
         if: ${{ matrix.db == 'mysql' }}
-        run: mysql -h 127.0.0.1 -e "GRANT ALL ON \`test\\_patchwork%\`.* to 'patchwork'@'%';" -uroot -proot
+        run: |
+          mysql -h 127.0.0.1 -e "GRANT ALL ON \`test\\_${DATABASE_NAME}%\`.* to '${DATABASE_USER}'@'%';" \
+              -uroot -p${MYSQL_ROOT_PASSWORD}
       - name: Run unit tests (via tox)
         run: tox
-        env:
-          PW_TEST_DB_TYPE: "${{ matrix.db }}"
-          PW_TEST_DB_USER: "patchwork"
-          PW_TEST_DB_PASS: "patchwork"
-          PW_TEST_DB_HOST: "127.0.0.1"
   docs:
     name: Build docs
     runs-on: ubuntu-latest
diff --git a/docker-compose-pg.yml b/docker-compose-pg.yml
index 44c49fb..7398929 100644
--- a/docker-compose-pg.yml
+++ b/docker-compose-pg.yml
@@ -24,9 +24,9 @@ services:
     environment:
       - UID
       - GID
-      - PW_TEST_DB_HOST=db
-      - PW_TEST_DB_PORT=5432
-      - PW_TEST_DB_TYPE=postgres
-      - PW_TEST_DB_USER=postgres
-      - PW_TEST_DB_PASS=password
-      - PGPASSWORD=password
+      - DATABASE_TYPE=postgres
+      - DATABASE_HOST=db
+      - DATABASE_PORT=5432
+      - DATABASE_NAME=patchwork
+      - DATABASE_USER=postgres
+      - DATABASE_PASSWORD=password
diff --git a/docker-compose.yml b/docker-compose.yml
index 1d49c51..103f19f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
     volumes:
       - ./tools/docker/db/data:/var/lib/mysql
     environment:
-      - MYSQL_ROOT_PASSWORD=password
+      - MYSQL_ROOT_PASSWORD=root
       - MYSQL_USER=patchwork
       - MYSQL_PASSWORD=password
 
@@ -26,5 +26,9 @@ services:
     environment:
       - UID
       - GID
-      - PW_TEST_DB_HOST=db
-      - PW_TEST_DB_PORT=3306
+      # skip DATABASE_TYPE explicitly as mysql should be the default type.
+      - DATABASE_HOST=db
+      - DATABASE_PORT=3306
+      - DATABASE_USER=patchwork
+      - DATABASE_PASSWORD=password
+      - MYSQL_ROOT_PASSWORD=root
diff --git a/docs/deployment/installation.rst b/docs/deployment/installation.rst
index 1abf187..14ad746 100644
--- a/docs/deployment/installation.rst
+++ b/docs/deployment/installation.rst
@@ -64,7 +64,7 @@ used to ease deployment:
   Username that the Patchwork web application will access the database with. We
   will use ``www-data``, for reasons described later in this guide.
 
-``DATABASE_PASS=``
+``DATABASE_PASSWORD=``
   Password that the Patchwork web application will access the database with. As
   we're going to use *peer* authentication (more on this later), this will be
   unset.
@@ -323,7 +323,7 @@ Once done, we should be able to check that all requirements are met using the
       $ sudo -u www-data \
           --preserve-env=DATABASE_NAME \
           --preserve-env=DATABASE_USER \
-          --preserve-env=DATABASE_PASS \
+          --preserve-env=DATABASE_PASSWORD \
           --preserve-env=DATABASE_HOST \
           --preserve-env=DATABASE_PORT \
           --preserve-env=STATIC_ROOT \
diff --git a/docs/development/installation.rst b/docs/development/installation.rst
index d16177c..3507f43 100644
--- a/docs/development/installation.rst
+++ b/docs/development/installation.rst
@@ -316,7 +316,7 @@ if necessary:
 
    The ``patchwork`` username and ``password`` password are the defaults
    expected by the provided ``dev`` settings files. If using something
-   different, export the ``PW_TEST_DB_USER`` and ``PW_TEST_DB_PASS`` variables
+   different, export the ``DATABASE_USER`` and ``DATABASE_PASSWORD`` variables
    described in the :ref:`Environment Variables <dev-envvar>` section below.
    Alternatively, you can create your own settings file with these variables
    hardcoded and change the value of ``DJANGO_SETTINGS_MODULE`` as described
@@ -470,17 +470,16 @@ __ https://django-dbbackup.readthedocs.io/en/stable/
 Environment Variables
 ---------------------
 
-The following environment variables are available to configure settings when
-using the provided ``dev`` settings file.
+The following environment variables are available to configure settings.
 
-``PW_TEST_DB_NAME=patchwork``
+``DATABASE_NAME=patchwork``
   Name of the database
 
-``PW_TEST_DB_USER=patchwork``
+``DATABASE_USER=patchwork``
   Username to access the database with
 
-``PW_TEST_DB_PASS=password``
+``DATABASE_PASSWORD=password``
   Password to access the database with<
 
-``PW_TEST_DB_TYPE=mysql``
+``DATABASE_TYPE=mysql``
   Type of database to use. Options: ``mysql``, ``postgres``
diff --git a/lib/uwsgi/patchwork.ini b/lib/uwsgi/patchwork.ini
index 95a0613..dbff508 100644
--- a/lib/uwsgi/patchwork.ini
+++ b/lib/uwsgi/patchwork.ini
@@ -10,7 +10,7 @@ url = /
 # env = DJANGO_SECRET_KEY=
 # env = DATABASE_NAME=
 # env = DATABASE_USER=
-# env = DATABASE_PASS=
+# env = DATABASE_PASSWORD=
 # env = DATABASE_HOST=
 # env = DATABASE_PORT=
 # env = STATIC_ROOT=
diff --git a/patchwork/settings/base.py b/patchwork/settings/base.py
index ff14d91..5e200f0 100644
--- a/patchwork/settings/base.py
+++ b/patchwork/settings/base.py
@@ -105,6 +105,45 @@ STATICFILES_DIRS = [
     os.path.join(ROOT_DIR, 'htdocs'),
 ]
 
+# Database
+#
+# If you're using a postgres database, connecting over a local unix-domain
+# socket, then the following setting should work for you. Otherwise,
+# see https://docs.djangoproject.com/en/2.2/ref/settings/#databases
+
+if os.getenv('DATABASE_TYPE', None) == 'postgres':
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.postgresql_psycopg2',
+            'HOST': os.environ.get('DATABASE_HOST', 'localhost'),
+            'PORT': os.environ.get('DATABASE_PORT', ''),
+            'NAME': os.environ.get('DATABASE_NAME', 'patchwork'),
+            'USER': os.environ.get('DATABASE_USER', 'patchwork'),
+            'PASSWORD': os.environ.get('DATABASE_PASSWORD', 'password'),
+        },
+    }
+elif os.getenv('DATABASE_TYPE', None) == 'sqlite3':
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': os.environ.get('DATABASE_NAME', ''),
+        },
+    }
+else:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.mysql',
+            'HOST': os.getenv('DATABASE_HOST', 'localhost'),
+            'PORT': os.getenv('DATABASE_PORT', ''),
+            'NAME': os.getenv('DATABASE_NAME', 'patchwork'),
+            'USER': os.getenv('DATABASE_USER', 'patchwork'),
+            'PASSWORD': os.getenv('DATABASE_PASSWORD', 'password'),
+            'TEST': {
+                'CHARSET': 'utf8',
+            },
+        },
+    }
+
 #
 # Third-party application settings
 #
diff --git a/patchwork/settings/dev.py b/patchwork/settings/dev.py
index 2b7fc95..cb4cb19 100644
--- a/patchwork/settings/dev.py
+++ b/patchwork/settings/dev.py
@@ -7,8 +7,6 @@ Design based on:
     http://www.revsys.com/blog/2014/nov/21/recommended-django-project-layout/
 """
 
-import os
-
 from .base import *  # noqa
 
 try:
@@ -36,30 +34,8 @@ SECRET_KEY = '00000000000000000000000000000000000000000000000000'  # noqa
 
 DEBUG = True
 
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.mysql',
-        'HOST': os.getenv('PW_TEST_DB_HOST', 'localhost'),
-        'PORT': os.getenv('PW_TEST_DB_PORT', ''),
-        'USER': os.getenv('PW_TEST_DB_USER', 'patchwork'),
-        'PASSWORD': os.getenv('PW_TEST_DB_PASS', 'password'),
-        'NAME': os.getenv('PW_TEST_DB_NAME', 'patchwork'),
-        'TEST': {
-            'CHARSET': 'utf8',
-        },
-    },
-}
-
-if os.getenv('PW_TEST_DB_TYPE', None) == 'postgres':
-    DATABASES['default']['ENGINE'] = 'django.db.backends.postgresql_psycopg2'
-    DATABASES['default']['HOST'] = os.getenv('PW_TEST_DB_HOST', '')
-elif os.getenv('PW_TEST_DB_TYPE', None) == 'sqlite':
-    DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
-    DATABASES['default']['NAME'] = '/dev/shm/patchwork.test.db.sqlite3'
-    del DATABASES['default']['HOST']
-    del DATABASES['default']['PORT']
-    del DATABASES['default']['USER']
-    del DATABASES['default']['PASSWORD']
+if DATABASES['default']['ENGINE'] == 'mysql':  # noqa: F405
+    DATABASES['default']['TEST'] = {'CHARSET': 'utf8'}  # noqa: F405
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
diff --git a/patchwork/settings/production.example.py b/patchwork/settings/production.example.py
index caaf429..ff05629 100644
--- a/patchwork/settings/production.example.py
+++ b/patchwork/settings/production.example.py
@@ -48,23 +48,6 @@ ADMINS = (
     # ('Jeremy Kerr', 'jk at ozlabs.org'),
 )
 
-# Database
-#
-# If you're using a postgres database, connecting over a local unix-domain
-# socket, then the following setting should work for you. Otherwise,
-# see https://docs.djangoproject.com/en/2.2/ref/settings/#databases
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': os.environ.get('DATABASE_NAME', ''),
-        'USER': os.environ.get('DATABASE_USER', ''),
-        'PASSWORD': os.environ.get('DATABASE_PASSWORD', ''),
-        'HOST': os.environ.get('DATABASE_HOST', ''),
-        'PORT': os.environ.get('DATABASE_PORT', ''),
-    },
-}
-
 #
 # Static files settings
 # https://docs.djangoproject.com/en/2.2/ref/settings/#static-files
diff --git a/tools/docker/entrypoint.sh b/tools/docker/entrypoint.sh
index 5450a53..faa36b3 100755
--- a/tools/docker/entrypoint.sh
+++ b/tools/docker/entrypoint.sh
@@ -1,45 +1,62 @@
 #!/bin/bash
 set -euo pipefail
 
-PW_TEST_DB_TYPE=${PW_TEST_DB_TYPE:-mysql}
+export DATABASE_HOST=${DATABASE_HOST:-}
+export DATABASE_PORT=${DATABASE_PORT:-}
+export DATABASE_NAME=${DATABASE_NAME:-patchwork}
+export DATABASE_USER=${DATABASE_USER:-patchwork}
+export DATABASE_PASSWORD=${DATABASE_PASSWORD:-password}
+
+case "${DATABASE_TYPE:-}" in
+postgres)
+    export PGPORT=${DATABASE_PORT}
+    export PGPASSWORD=${DATABASE_PASSWORD}
+    psql_args=( ${DATABASE_HOST:+--host=${DATABASE_HOST}} "--username=${DATABASE_USER}" )
+    ;;
+*)
+    export DATABASE_TYPE=mysql
+    mysql_args=( ${DATABASE_HOST:+--host=${DATABASE_HOST}} ${DATABASE_PORT:+--port=${DATABASE_PORT}} "--user=${DATABASE_USER}" "--password=${DATABASE_PASSWORD}" )
+    mysql_root_args=( ${DATABASE_HOST:+--host=${DATABASE_HOST}} ${DATABASE_PORT:+--port=${DATABASE_PORT}} "--user=root" "--password=${MYSQL_ROOT_PASSWORD:-}" )
+    ;;
+esac
 
 # functions
 
 test_db_connection() {
-    if [ ${PW_TEST_DB_TYPE} = "postgres" ]; then
-        echo ';' | psql -h $PW_TEST_DB_HOST -U postgres 2> /dev/null > /dev/null
+    if [ ${DATABASE_TYPE} = "postgres" ]; then
+        echo ';' | psql "${psql_args[@]}" 2> /dev/null > /dev/null
     else
-        mysqladmin -h $PW_TEST_DB_HOST -u patchwork --password=password ping > /dev/null 2> /dev/null
+        mysqladmin "${mysql_root_args[@]}" ping > /dev/null 2> /dev/null
     fi
 }
 
 test_database() {
-    if [ ${PW_TEST_DB_TYPE} = "postgres" ]; then
-        echo ';' | psql -h $PW_TEST_DB_HOST -U postgres patchwork 2> /dev/null
+    if [ ${DATABASE_TYPE} = "postgres" ]; then
+        echo ';' | psql "${psql_args[@]}" "${DATABASE_NAME}" 2> /dev/null
     else
-        echo ';' | mysql -h $PW_TEST_DB_HOST -u patchwork -ppassword patchwork 2> /dev/null
+        echo ';' | mysql "${mysql_args[@]}" "${DATABASE_NAME}" 2> /dev/null
     fi
 }
 
 reset_data_mysql() {
-    mysql -uroot -ppassword -h $PW_TEST_DB_HOST << EOF
-DROP DATABASE IF EXISTS patchwork;
-CREATE DATABASE patchwork CHARACTER SET utf8;
-GRANT ALL ON patchwork.* TO 'patchwork' IDENTIFIED BY 'password';
-GRANT ALL ON \`test\\_patchwork%\`.* to 'patchwork'@'%';
+    mysql "${mysql_root_args[@]}" << EOF
+DROP DATABASE IF EXISTS ${DATABASE_NAME};
+CREATE DATABASE ${DATABASE_NAME} CHARACTER SET utf8;
+GRANT ALL ON ${DATABASE_NAME}.* TO '${DATABASE_USER}' IDENTIFIED BY '${DATABASE_PASSWORD}';
+GRANT ALL ON \`test\\_${DATABASE_NAME}%\`.* to '${DATABASE_USER}'@'%';
 FLUSH PRIVILEGES;
 EOF
 }
 
 reset_data_postgres() {
-    psql -h $PW_TEST_DB_HOST -U postgres <<EOF
-DROP DATABASE IF EXISTS patchwork;
-CREATE DATABASE patchwork WITH ENCODING = 'UTF8';
+    psql "${psql_args[@]}" <<EOF
+DROP DATABASE IF EXISTS ${DATABASE_NAME};
+CREATE DATABASE ${DATABASE_NAME} WITH ENCODING = 'UTF8';
 EOF
 }
 
 reset_data() {
-    if [ x${PW_TEST_DB_TYPE} = x"postgres" ]; then
+    if [ x${DATABASE_TYPE} = x"postgres" ]; then
         reset_data_postgres
     else
         reset_data_mysql
@@ -87,7 +104,7 @@ set -e
 
 # check if db is connected
 if ! test_db_connection; then
-    echo "The database seems not to be connected, or the patchwork user is broken"
+    echo "The database seems not to be connected, or the ${DATABASE_USER} user is broken"
     echo "MySQL/Postgres may still be starting. Waiting 5 seconds."
     sleep 5
     if ! test_db_connection; then
diff --git a/tox.ini b/tox.ini
index 3275600..374cbb9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,8 +25,8 @@ setenv =
     py{37,38,39}: PYTHONWARNINGS = once
 passenv =
     http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
-    PW_TEST_DB_TYPE PW_TEST_DB_USER PW_TEST_DB_PASS PW_TEST_DB_HOST
-    PW_TEST_DB_PORT PW_TEST_DB_NAME DJANGO_TEST_PROCESSES
+    DATABASE_TYPE DATABASE_USER DATABASE_PASSWORD DATABASE_HOST
+    DATABASE_PORT DATABASE_NAME DJANGO_TEST_PROCESSES
 commands =
     python {toxinidir}/manage.py test --noinput --parallel -- {posargs:patchwork}
 
-- 
2.32.0



More information about the Patchwork mailing list