root: automate system migrations, move docker to lifecycle folder

Dockerfile

@@ -11,27 +11,22 @@ RUN pip install pipenv && \
 
 FROM python:3.8-slim-buster
 
-COPY --from=locker /app/requirements.txt /app/
-COPY --from=locker /app/requirements-dev.txt /app/
-
-WORKDIR /app/
+WORKDIR /
+COPY --from=locker /app/requirements.txt /
+COPY --from=locker /app/requirements-dev.txt /
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends postgresql-client-11 build-essential && \
     rm -rf /var/lib/apt/ && \
-    pip install -r requirements.txt  --no-cache-dir && \
+    pip install -r /requirements.txt  --no-cache-dir && \
     apt-get remove --purge -y build-essential && \
     apt-get autoremove --purge && \
-    adduser --system --no-create-home --uid 1000 --group --home /app passbook
+    adduser --system --no-create-home --uid 1000 --group --home /passbook passbook
 
-COPY ./passbook/ /app/passbook
-COPY ./manage.py /app/
-COPY ./docker/gunicorn.conf.py /app/
-COPY ./docker/bootstrap.sh /bootstrap.sh
-COPY ./docker/wait_for_db.py /app/wait_for_db.py
-
-WORKDIR /app/
+COPY ./passbook/ /passbook
+COPY ./manage.py /
+COPY ./lifecycle/ /lifecycle
 
 USER passbook
 
-ENTRYPOINT [ "/bootstrap.sh" ]
+ENTRYPOINT [ "/lifecycle/bootstrap.sh" ]

README.md

@@ -25,7 +25,7 @@ wget https://raw.githubusercontent.com/BeryJu/passbook/master/docker-compose.yml
 # export PG_PASS=$(pwgen 40 1)
 docker-compose pull
 docker-compose up -d
-docker-compose exec server ./manage.py migrate
+docker-compose run --rm server migrate
 ```
 
 For bigger setups, there is a Helm Chart in the `helm/` directory. This is documented [here](https://passbook.beryju.org//installation/kubernetes/)

docker/bootstrap.sh

@@ -1,10 +0,0 @@
-#!/bin/bash -e
-/app/wait_for_db.py
-printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@"
-if [[ "$1" == "server" ]]; then
-    gunicorn -c gunicorn.conf.py passbook.root.asgi:application
-elif [[ "$1" == "worker" ]]; then
-    celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule
-else
-    ./manage.py "$@"
-fi

e2e/test_flows_enroll.py

@@ -2,13 +2,13 @@
 from time import sleep
 
 from django.test import override_settings
+from docker import DockerClient, from_env
+from docker.models.containers import Container
+from docker.types import Healthcheck
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
 from e2e.utils import USER, SeleniumTestCase
 from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding
 from passbook.policies.expression.models import ExpressionPolicy

e2e/test_provider_oauth2_github.py

@@ -1,13 +1,13 @@
 """test OAuth Provider flow"""
 from time import sleep
 
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from structlog import get_logger
-
 from docker import DockerClient, from_env
 from docker.models.containers import Container
 from docker.types import Healthcheck
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from structlog import get_logger
+
 from e2e.utils import USER, SeleniumTestCase
 from passbook.core.models import Application
 from passbook.flows.models import Flow

e2e/test_provider_oauth2_oidc.py

@@ -1,14 +1,14 @@
 """test OAuth2 OpenID Provider flow"""
 from time import sleep
 
+from docker import DockerClient, from_env
+from docker.models.containers import Container
+from docker.types import Healthcheck
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
 from e2e.utils import USER, SeleniumTestCase
 from passbook.core.models import Application
 from passbook.crypto.models import CertificateKeyPair

e2e/test_provider_saml.py

@@ -1,13 +1,13 @@
 """test SAML Provider flow"""
 from time import sleep
 
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from structlog import get_logger
-
 from docker import DockerClient, from_env
 from docker.models.containers import Container
 from docker.types import Healthcheck
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from structlog import get_logger
+
 from e2e.utils import USER, SeleniumTestCase
 from passbook.core.models import Application
 from passbook.crypto.models import CertificateKeyPair

e2e/test_source_saml.py

@@ -1,14 +1,14 @@
 """test SAML Source"""
 from time import sleep
 
+from docker import DockerClient, from_env
+from docker.models.containers import Container
+from docker.types import Healthcheck
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
 from e2e.utils import SeleniumTestCase
 from passbook.crypto.models import CertificateKeyPair
 from passbook.flows.models import Flow

e2e/test_sources_oauth.py

@@ -2,15 +2,15 @@
 from os.path import abspath
 from time import sleep
 
+from docker import DockerClient, from_env
+from docker.models.containers import Container
+from docker.types import Healthcheck
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 from yaml import safe_dump
 
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
 from e2e.utils import SeleniumTestCase
 from passbook.flows.models import Flow
 from passbook.providers.oauth2.generators import generate_client_secret

lifecycle/bootstrap.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+python -m lifecycle.wait_for_db
+printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@"
+if [[ "$1" == "server" ]]; then
+    gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application
+elif [[ "$1" == "worker" ]]; then
+    celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule
+elif [[ "$1" == "migrate" ]]; then
+    # Run system migrations first, run normal migrations after
+    python -m lifecycle.migrate
+    python -m manage migrate
+else
+    python -m manage "$@"
+fi

lifecycle/migrate.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""System Migration handler"""
+from importlib.util import module_from_spec, spec_from_file_location
+from inspect import getmembers, isclass
+from pathlib import Path
+from typing import Any
+
+from psycopg2 import connect
+from structlog import get_logger
+
+from passbook.lib.config import CONFIG
+
+LOGGER = get_logger()
+
+
+class BaseMigration:
+    """Base System Migration"""
+
+    cur: Any
+    con: Any
+
+    def __init__(self, cur: Any, con: Any):
+        self.cur = cur
+        self.con = con
+
+    def needs_migration(self) -> bool:
+        """Return true if Migration needs to be run"""
+        return False
+
+    def run(self):
+        """Run the actual migration"""
+
+
+if __name__ == "__main__":
+
+    conn = connect(
+        dbname=CONFIG.y("postgresql.name"),
+        user=CONFIG.y("postgresql.user"),
+        password=CONFIG.y("postgresql.password"),
+        host=CONFIG.y("postgresql.host"),
+    )
+    curr = conn.cursor()
+
+    for migration in Path(__file__).parent.absolute().glob("system_migrations/*.py"):
+        spec = spec_from_file_location("lifecycle.system_migrations", migration)
+        mod = module_from_spec(spec)
+        # pyright: reportGeneralTypeIssues=false
+        spec.loader.exec_module(mod)
+
+        for _, sub in getmembers(mod, isclass):
+            migration = sub(curr, conn)
+            if migration.needs_migration():
+                LOGGER.info("Migration needs to be applied", migration=sub)
+                migration.run()
+                LOGGER.info("Migration finished applying", migration=sub)

lifecycle/system_migrations/to_0_10.py

@@ -0,0 +1,50 @@
+from os import system
+
+from lifecycle.migrate import BaseMigration
+
+SQL_STATEMENT = """delete from django_migrations where app = 'passbook_stages_prompt';
+drop table passbook_stages_prompt_prompt cascade;
+drop table passbook_stages_prompt_promptstage cascade;
+drop table passbook_stages_prompt_promptstage_fields;
+drop table corsheaders_corsmodel cascade;
+drop table oauth2_provider_accesstoken cascade;
+drop table oauth2_provider_grant cascade;
+drop table oauth2_provider_refreshtoken cascade;
+drop table oidc_provider_client cascade;
+drop table oidc_provider_client_response_types cascade;
+drop table oidc_provider_code cascade;
+drop table oidc_provider_responsetype cascade;
+drop table oidc_provider_rsakey cascade;
+drop table oidc_provider_token cascade;
+drop table oidc_provider_userconsent cascade;
+drop table passbook_providers_app_gw_applicationgatewayprovider cascade;
+delete from django_migrations where app = 'passbook_flows' and name = '0008_default_flows';
+delete from django_migrations where app = 'passbook_flows' and name = '0009_source_flows';
+delete from django_migrations where app = 'passbook_flows' and name = '0010_provider_flows';
+delete from django_migrations where app = 'passbook_stages_password' and
+name = '0002_passwordstage_change_flow';"""
+
+
+class To010Migration(BaseMigration):
+    def needs_migration(self) -> bool:
+        self.cur.execute(
+            "select * from information_schema.tables where table_name='oidc_provider_client'"
+        )
+        return bool(self.cur.rowcount)
+
+    def system_crit(self, command):
+        retval = system(command)  # nosec
+        if retval != 0:
+            raise Exception("Migration error")
+
+    def run(self):
+        self.cur.execute(SQL_STATEMENT)
+        self.con.commit()
+        self.system_crit("./manage.py migrate passbook_stages_prompt")
+        self.system_crit("./manage.py migrate passbook_flows 0008_default_flows --fake")
+        self.system_crit("./manage.py migrate passbook_flows 0009_source_flows --fake")
+        self.system_crit(
+            "./manage.py migrate passbook_flows 0010_provider_flows --fake"
+        )
+        self.system_crit("./manage.py migrate passbook_flows")
+        self.system_crit("./manage.py migrate passbook_stages_password --fake")

swagger.yaml

@@ -5249,7 +5249,7 @@ paths:
       tags:
         - stages
     parameters: []
-  /stages/prompt/stages/{pbm_uuid}/:
+  /stages/prompt/stages/{stage_uuid}/:
     get:
       operationId: stages_prompt_stages_read
       description: PromptStage Viewset
@@ -5303,7 +5303,7 @@ paths:
       tags:
         - stages
     parameters:
-      - name: pbm_uuid
+      - name: stage_uuid
         in: path
         description: A UUID string identifying this Prompt Stage.
         required: true
@@ -7463,7 +7463,7 @@ definitions:
     type: object
     properties:
       pk:
-        title: Pbm uuid
+        title: Stage uuid
         type: string
         format: uuid
         readOnly: true
@@ -7477,6 +7477,12 @@ definitions:
           type: string
           format: uuid
         uniqueItems: true
+      validation_policies:
+        type: array
+        items:
+          type: string
+          format: uuid
+        uniqueItems: true
   UserDeleteStage:
     required:
       - name