vauxite-build/ostree-engine

#!/usr/bin/env python3

"""ostree-engine

Usage:
    ostree-engine ([--source-branch=BRANCH] [--source-url=URL] | --no-download-sources)
                  [--ostree-branch=REF] [--treefile=TREEFILE]
                  [--dest-repo=PATH] [--gpg-passfile=PATH]
                  [--no-deploy] [--no-clean]
    ostree-engine --version

Options:
    --no-deploy                 Do not deploy resulting ostree repo to web server root.
    --no-clean                  Do not clean the working directory, i.e. .cache, .build-repo, .source-repo, .tmp
    --no-download-sources       Do not download source repo.
    --source-url=URL            URL for source repo. [default: https://pagure.io/workstation-ostree-config]
    --source-branch=BRANCH      Branch of the source repo.
    --treefile=TREEFILE         YAML or JSON treefile to use for rpm-ostree.
    --ostree-branch=REF         Name of the ref branch.
    --dest-repo=PATH            Local or remote filesystem destination for rsync-repos. Usually a web server root. [default: /srv/repo]
    --gpg-passfile=PATH         Path to JSON file containing GPG key-id and passphrase (for auto signing commits).
    --version                   Print version information.
"""

import datetime as dt
import json
import os
import shlex
import shutil
import subprocess
import sys
from docopt import docopt
from glob import glob
from pathlib import Path

if os.geteuid() != 0:
    exit("Please run this script with sudo")

BASE_DIR = Path("/var/local/vauxite")
OSTREE_FILES_DIR = BASE_DIR.joinpath("src")
CACHE_DIR = BASE_DIR.joinpath(".cache")
BUILD_REPO = BASE_DIR.joinpath(".build-repo")
SOURCE_REPO = BASE_DIR.joinpath(".source-repo")
DEPLOY_REPO = BASE_DIR.joinpath(".deploy-repo")
WK_DIR = BASE_DIR.joinpath(".tmp")


def print_log(msg: str):
    if sys.stdout.isatty():
        log_date = dt.datetime.now().isoformat(" ", "seconds")
        print("%s: %s" % (log_date, msg))
    else:
        print(msg)


def handle_err():
    print_log("ERROR:")
    print(f"{sys.exc_info()[0]}")
    print(f"{sys.exc_info()[1]}")
    print(f"{sys.exc_info()[2]}")
    exit(1)


def clean_wk_dir():
    try:
        print_log("Clean working directory")
        shutil.rmtree(CACHE_DIR)
        if not args.get("--no-download-sources"):
            shutil.rmtree(SOURCE_REPO)
        shutil.rmtree(WK_DIR)
        for dir in glob("/tmp/rpmostree*", recursive=True):
            shutil.rmtree(dir)
    except FileNotFoundError as ferr:
        pass
    except Exception:
        handle_err()


def run_proc(cmd: str, capture_output=False) -> subprocess.CompletedProcess:
    try:
        if capture_output:
            return subprocess.run(shlex.split(cmd), capture_output=True, text=True)

        return subprocess.run(shlex.split(cmd), check=True, text=True)
    except subprocess.CalledProcessError:
        handle_err()


def prepare_build_env():
    if not args.get("--no-clean"):
        clean_wk_dir()

    print_log("Ensure CACHE_DIR exists")
    CACHE_DIR.mkdir(exist_ok=True)

    print_log("Ensure WK_DIR exists")
    WK_DIR.mkdir(exist_ok=True)

    print_log("Remove previous BUILD_REPO if it exists")
    shutil.rmtree(BUILD_REPO, ignore_errors=True)

    print_log("Initialize ostree repo in bare-user mode")
    run_proc(f"ostree --repo={BUILD_REPO} init --mode=bare-user")

    if not DEPLOY_REPO.exists():
        print_log("Deploy repo not found; initialize new deploy repo in archive mode")
        run_proc(f"ostree --repo={BUILD_REPO} init --mode=archive")
    else:
        print_log("Pull existing deploy repo into local build repo")
        run_proc(
            f"ostree --repo={BUILD_REPO} pull-local --depth=2 {DEPLOY_REPO} {args.get("--ostree-branch")}"
        )

    if not SOURCE_REPO.exists():
        print_log(
            f"Clone branch {args.get("--source-branch")} of {args.get("--source-url")} into SOURCE_REPO"
        )
        run_proc(f"git clone -b {args.get("--source-branch")} {args.get("--source-url")} {SOURCE_REPO}")

    print_log("Copy SOURCE_REPO contents into WK_DIR")
    run_proc(f"rsync -aAX {SOURCE_REPO}/ {WK_DIR}")

    print_log("Remove upstream xfce-desktop-pkgs.yaml from WK_DIR")
    WK_DIR.joinpath("xfce-desktop-pkgs.yaml").unlink(missing_ok=True)

    print_log("Copy OSTREE_FILES_DIR contents into WK_DIR")
    run_proc(f"rsync -aAX {OSTREE_FILES_DIR}/ {WK_DIR}")


def compose_ostree():
    print_log("Compose ostree")
    time_fmt = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H%M%S")
    run_proc(
        subprocess.list2cmdline(
            [
                "rpm-ostree",
                "compose",
                "tree",
                "--unified-core",
                f"--cachedir={CACHE_DIR}",
                f"--repo={BUILD_REPO}",
                f"--add-metadata-string=Build={time_fmt}",
                WK_DIR.joinpath(args.get("--treefile")),
            ]
        )
    )


def sign_commit():
    commit_id = subprocess.run(["ostree", f"--repo={DEPLOY_REPO}", "rev-parse", args.get("--ostree-branch")], capture_output=True, text=True)

    with open(args.get("--gpg-passfile"), "r") as json_file:
        gpg_data = json.loads(json_file.read())

    print_log(f"Signing rpm-ostree commit {commit_id.stdout} with GPG key-id {gpg_data.get("gpg-id")}")

    gpg_cmd = f"echo {gpg_data.get("passphrase")} | gpg --batch --always-trust --yes --passphrase-fd 0 --pinentry-mode=loopback -s $(mktemp)"
    run_gpg_cmd = subprocess.Popen(gpg_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    output = run_gpg_cmd.communicate()[0]
    if output:
        print_log(f"Error unlocking GPG keyring: {output}")
        exit(99)

    run_proc(f"ostree --repo={DEPLOY_REPO} gpg-sign {commit_id.stdout} {gpg_data.get("gpg-id")}")


def prepare_deploy():
    print_log("Prune refs older than 30 days")
    run_proc(
        f"ostree --repo={BUILD_REPO} prune --refs-only --keep-younger-than='30 days ago'"
    )

    print_log("Pull new ostree commit into DEPLOY_REPO")
    run_proc(
        f"ostree --repo={DEPLOY_REPO} pull-local --depth=1 {BUILD_REPO} {args.get("--ostree-branch")}"
    )

    print_log("Remove local build repo")
    shutil.rmtree(BUILD_REPO, ignore_errors=True)

    print_log("Check filesystem for errors")
    run_proc(f"ostree --repo={DEPLOY_REPO} fsck {args.get("--ostree-branch")}")


def generate_deltas():
    print_log("Check if main ref has parent")
    check_parent = run_proc(
        f"ostree --repo={DEPLOY_REPO} show {args.get("--ostree-branch")}", capture_output=True
    )

    if not check_parent.stderr:
        print_log("Generate static delta from main ref's parent")
        run_proc(
            f"ostree --repo={DEPLOY_REPO} static-delta generate {args.get("--ostree-branch")}"
        )

        print_log("Check if main ref's parent has parent")
        check_gparent = run_proc(
            f"ostree --repo={DEPLOY_REPO} show {args.get("--ostree-branch")}^^",
            capture_output=True,
        )

        if not check_gparent.stderr:
            print_log("Generate static delta from parent of main ref's parent")
            run_proc(
                f"ostree --repo={DEPLOY_REPO} static-delta generate --from={args.get("--ostree-branch")}^^ --to={args.get("--ostree-branch")}"
            )
        else:
            print_log("Main ref's parent has no parent. No deltas generated.")
    else:
        print_log("Main ref has no parent. No deltas generated.")


def update_summary():
    print_log("Update summary file of DEPLOY_REPO")
    run_proc(f"ostree --repo={DEPLOY_REPO} summary -u")


def deploy_repo():
    print_log("Deploying repo to web server root")
    if DEPLOY_REPO.exists():
        run_proc(f"{BASE_DIR}/rsync-repos --src {DEPLOY_REPO} --dest {args.get("--dest-repo")}")
    else:
        print_log("DEPLOY_REPO not found. Not deploying to web server")


if __name__ == "__main__":
    args = docopt(__doc__, help=True, version="ostree-engine 0.1.0")
    prepare_build_env()
    compose_ostree()
    prepare_deploy()
    generate_deltas()
    if args.get("--gpg-passfile"):
        sign_commit()
    update_summary()
    if not args.get("--no-deploy"):
        deploy_repo()