mirror of
https://codeberg.org/hyperreal/vauxite-build
synced 2024-11-25 12:23:42 +01:00
Rewrite ostree-engine in Bash
This commit is contained in:
Jeffrey Serio
parent
ca05f01dd6
commit
c412151c15
338
ostree-engine
338
ostree-engine
@ -1,237 +1,157 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
"""ostree-engine
|
set -euo pipefail
|
||||||
|
|
||||||
Usage:
|
SOURCE_BRANCH="f39"
|
||||||
ostree-engine ([--source-branch=BRANCH] [--source-url=URL] | --no-download-sources)
|
SOURCE_URL="https://pagure.io/workstation-ostree-config"
|
||||||
[--ostree-branch=REF] [--treefile=TREEFILE]
|
OSTREE_BRANCH="vauxite/f39/x86_64/main"
|
||||||
[--dest-repo=PATH] [--gpg-passfile=PATH]
|
DEST_REPO="/srv/repo"
|
||||||
[--no-deploy] [--no-clean]
|
OSTREE_FILES_DIR="$(pwd)/src"
|
||||||
ostree-engine --version
|
CACHE_DIR="$(pwd)/.cache"
|
||||||
|
BUILD_REPO="$(pwd)/.build-repo"
|
||||||
|
SOURCE_REPO="$(pwd)/.source-repo"
|
||||||
|
TMP_WORK_DIR="$(pwd)/.tmp"
|
||||||
|
DEPLOY_REPO="$(pwd)/.deploy-repo"
|
||||||
|
TREEFILE="${TMP_WORK_DIR}/vauxite.json"
|
||||||
|
|
||||||
Options:
|
if [ "$(id -u)" != "0" ]; then
|
||||||
--no-deploy Do not deploy resulting ostree repo to web server root.
|
gum log --time datetime --level error "Please run build with sudo"
|
||||||
--no-clean Do not clean the working directory, i.e. .cache, .build-repo, .source-repo, .tmp
|
exit
|
||||||
--no-download-sources Do not download source repo.
|
fi
|
||||||
--source-url=URL URL for source repo. [default: https://pagure.io/workstation-ostree-config]
|
|
||||||
--source-branch=BRANCH Branch of the source repo.
|
|
||||||
--treefile=TREEFILE YAML or JSON treefile to use for rpm-ostree.
|
|
||||||
--ostree-branch=REF Name of the ref branch.
|
|
||||||
--dest-repo=PATH Local or remote filesystem destination for rsync-repos. Usually a web server root. [default: /srv/repo]
|
|
||||||
--gpg-passfile=PATH Path to JSON file containing GPG key-id and passphrase (for auto signing commits).
|
|
||||||
--version Print version information.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import datetime as dt
|
function log_struc_info() {
|
||||||
import json
|
gum log --time datetime --structured --level info "$@"
|
||||||
import os
|
}
|
||||||
import shlex
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from docopt import docopt
|
|
||||||
from glob import glob
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
if os.geteuid() != 0:
|
function log_info() {
|
||||||
exit("Please run this script with sudo")
|
gum log --time datetime --level info "$@"
|
||||||
|
}
|
||||||
|
|
||||||
BASE_DIR = Path("/var/local/vauxite")
|
function log_struc_error() {
|
||||||
OSTREE_FILES_DIR = BASE_DIR.joinpath("src")
|
gum log --time datetime --structured --level error "$@"
|
||||||
CACHE_DIR = BASE_DIR.joinpath(".cache")
|
}
|
||||||
BUILD_REPO = BASE_DIR.joinpath(".build-repo")
|
|
||||||
SOURCE_REPO = BASE_DIR.joinpath(".source-repo")
|
|
||||||
DEPLOY_REPO = BASE_DIR.joinpath(".deploy-repo")
|
|
||||||
WK_DIR = BASE_DIR.joinpath(".tmp")
|
|
||||||
|
|
||||||
|
# Clean working directory
|
||||||
|
log_struc_info "Clean cache directory" directory "${CACHE_DIR}"
|
||||||
|
rm -rf "${CACHE_DIR}"
|
||||||
|
|
||||||
def print_log(msg: str):
|
log_struc_info "Clean source repo" directory "${SOURCE_REPO}"
|
||||||
if sys.stdout.isatty():
|
rm -rf "${SOURCE_REPO}"
|
||||||
log_date = dt.datetime.now().isoformat(" ", "seconds")
|
|
||||||
print("%s: %s" % (log_date, msg))
|
|
||||||
else:
|
|
||||||
print(msg)
|
|
||||||
|
|
||||||
|
log_struc_info "Clean temporary working directory" directory "${TMP_WORK_DIR}"
|
||||||
|
rm -rf "${TMP_WORK_DIR}"
|
||||||
|
|
||||||
def handle_err():
|
log_struc_info "Clean /tmp/rpmostree*" files /tmp/rpmostree*
|
||||||
print_log("ERROR:")
|
rm -rf /tmp/rpmostree*
|
||||||
print(f"{sys.exc_info()[0]}")
|
|
||||||
print(f"{sys.exc_info()[1]}")
|
|
||||||
print(f"{sys.exc_info()[2]}")
|
|
||||||
exit(1)
|
|
||||||
|
|
||||||
|
log_struc_info "Clean build repo" directory "${BUILD_REPO}"
|
||||||
|
rm -rf "${BUILD_REPO}"
|
||||||
|
|
||||||
def clean_wk_dir():
|
# Prepare build env
|
||||||
try:
|
log_struc_info "Ensure cache directory exists" directory "${CACHE_DIR}"
|
||||||
print_log("Clean working directory")
|
mkdir -p "${CACHE_DIR}"
|
||||||
shutil.rmtree(CACHE_DIR)
|
|
||||||
if not args.get("--no-download-sources"):
|
|
||||||
shutil.rmtree(SOURCE_REPO)
|
|
||||||
shutil.rmtree(WK_DIR)
|
|
||||||
for dir in glob("/tmp/rpmostree*", recursive=True):
|
|
||||||
shutil.rmtree(dir)
|
|
||||||
except FileNotFoundError as ferr:
|
|
||||||
pass
|
|
||||||
except Exception:
|
|
||||||
handle_err()
|
|
||||||
|
|
||||||
|
log_struc_info "Ensure temporary working directory exists" directory "${TMP_WORK_DIR}"
|
||||||
|
mkdir -p "${TMP_WORK_DIR}"
|
||||||
|
|
||||||
def run_proc(cmd: str, capture_output=False) -> subprocess.CompletedProcess:
|
if [ -d "${DEPLOY_REPO}" ] && [ -n "$(ls "${DEPLOY_REPO}")" ]; then
|
||||||
try:
|
log_info "Deploy repo found. Initialize ostree repo in bare-user mode."
|
||||||
if capture_output:
|
if ! ostree --repo="${BUILD_REPO}" init --mode=bare-user; then
|
||||||
return subprocess.run(shlex.split(cmd), capture_output=True, text=True)
|
log_struc_error "Error initializing ostree repo in bare-user mode" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
return subprocess.run(shlex.split(cmd), check=True, text=True)
|
log_info "Pull existing deploy repo into local build repo"
|
||||||
except subprocess.CalledProcessError:
|
if ! ostree --repo="${BUILD_REPO}" pull-local --depth=2 "${DEPLOY_REPO}" "${OSTREE_BRANCH}"; then
|
||||||
handle_err()
|
log_struc_error "Error pulling existing deploy repo into local build repo" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log_info "Deploy repo not found. Initialize new deploy repo in archive mode."
|
||||||
|
if ! ostree --repo="${BUILD_REPO}" init --mode=archive; then
|
||||||
|
log_struc_error "Error initializing new deploy repo in archive mode" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_struc_info "Clone source repo" url "${SOURCE_URL}" branch "${SOURCE_BRANCH}" directory "${SOURCE_REPO}"
|
||||||
|
if ! git clone -b "${SOURCE_BRANCH}" "${SOURCE_URL}" "${SOURCE_REPO}"; then
|
||||||
|
log_struc_error "Error cloning source repo" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
def prepare_build_env():
|
log_struc_info "Copy contents of source repo into temporary work directory" source_repo "${SOURCE_REPO}" directory "${TMP_WORK_DIR}"
|
||||||
if not args.get("--no-clean"):
|
rsync -aAX "${SOURCE_REPO}"/ "${TMP_WORK_DIR}"
|
||||||
clean_wk_dir()
|
|
||||||
|
|
||||||
print_log("Ensure CACHE_DIR exists")
|
log_struc_info "Remove upstream xfce-desktop-pkgs.yaml from temporary work directory" file xfce-desktop-pkgs.yaml directory "${TMP_WORK_DIR}"
|
||||||
CACHE_DIR.mkdir(exist_ok=True)
|
rm -f "${TMP_WORK_DIR}"/xfce-desktop-pkgs.yaml
|
||||||
|
|
||||||
print_log("Ensure WK_DIR exists")
|
log_struc_info "Copy contents of ostree files directory into temporary work directory" source "${OSTREE_FILES_DIR}" dest "${TMP_WORK_DIR}"
|
||||||
WK_DIR.mkdir(exist_ok=True)
|
rsync -aAX "${OSTREE_FILES_DIR}"/ "${TMP_WORK_DIR}"
|
||||||
|
|
||||||
print_log("Remove previous BUILD_REPO if it exists")
|
# Compose ostree
|
||||||
shutil.rmtree(BUILD_REPO, ignore_errors=True)
|
METADATA_STR="$(date '+%Y-%m-%dT%H%M%S')"
|
||||||
|
log_struc_info "Compose ostree" cachedir "${CACHE_DIR}" repo "${BUILD_REPO}" metadata-string "${METADATA_STR}" treefile "${TREEFILE}"
|
||||||
|
if ! rpm-ostree compose tree --unified-core --cachedir="${CACHE_DIR}" --repo="${BUILD_REPO}" --add-metadata-string=Build="${METADATA_STR}" "${TREEFILE}"; then
|
||||||
|
log_struc_error "Error composing ostree" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
print_log("Initialize ostree repo in bare-user mode")
|
# Prepare deploy
|
||||||
run_proc(f"ostree --repo={BUILD_REPO} init --mode=bare-user")
|
log_info "Prune refs older than 30 days"
|
||||||
|
if ! ostree --repo="${BUILD_REPO}" prune --refs-only --keep-younger-than='30 days ago'; then
|
||||||
|
log_struc_error "Error pruning refs" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
if not DEPLOY_REPO.exists():
|
log_struc_info "Pull new ostree commit into deploy repo" deploy_repo "${DEPLOY_REPO}"
|
||||||
print_log("Deploy repo not found; initialize new deploy repo in archive mode")
|
if ! ostree --repo="${DEPLOY_REPO}" pull-local --depth=1 "${BUILD_REPO}" "${OSTREE_BRANCH}"; then
|
||||||
run_proc(f"ostree --repo={BUILD_REPO} init --mode=archive")
|
log_struc_error "Error pulling new ostree commit into deploy repo" status "$?"
|
||||||
else:
|
exit
|
||||||
print_log("Pull existing deploy repo into local build repo")
|
fi
|
||||||
run_proc(
|
|
||||||
f"ostree --repo={BUILD_REPO} pull-local --depth=2 {DEPLOY_REPO} {args.get("--ostree-branch")}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if not SOURCE_REPO.exists():
|
log_struc_info "Remove local build repo" build_repo "${BUILD_REPO}"
|
||||||
print_log(
|
rm -rf "${BUILD_REPO}"
|
||||||
f"Clone branch {args.get("--source-branch")} of {args.get("--source-url")} into SOURCE_REPO"
|
|
||||||
)
|
|
||||||
run_proc(f"git clone -b {args.get("--source-branch")} {args.get("--source-url")} {SOURCE_REPO}")
|
|
||||||
|
|
||||||
print_log("Copy SOURCE_REPO contents into WK_DIR")
|
log_struc_info "Check filesystem for errors" deploy_repo "${DEPLOY_REPO}" ostree_branch "${OSTREE_BRANCH}"
|
||||||
run_proc(f"rsync -aAX {SOURCE_REPO}/ {WK_DIR}")
|
if ! ostree --repo="${DEPLOY_REPO}" fsck "${OSTREE_BRANCH}"; then
|
||||||
|
log_struc_error "Error checking filesystem for errors" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
print_log("Remove upstream xfce-desktop-pkgs.yaml from WK_DIR")
|
# Generate deltas
|
||||||
WK_DIR.joinpath("xfce-desktop-pkgs.yaml").unlink(missing_ok=True)
|
log_info "Check if main ref has parent"
|
||||||
|
if ! ostree --repo="${DEPLOY_REPO}" show "${OSTREE_BRANCH}"; then
|
||||||
|
log_info "Main ref has no parent. No deltas will be generated."
|
||||||
|
else
|
||||||
|
log_info "Generate static delta from main ref's parent"
|
||||||
|
if ! ostree --repo="${DEPLOY_REPO}" static-delta generate "${OSTREE_BRANCH}"; then
|
||||||
|
log_struc_error "Error generating static delta from main ref's parent" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
print_log("Copy OSTREE_FILES_DIR contents into WK_DIR")
|
log_info "Check if main ref has grandparent"
|
||||||
run_proc(f"rsync -aAX {OSTREE_FILES_DIR}/ {WK_DIR}")
|
if ! ostree --repo="${DEPLOY_REPO}" show "${OSTREE_BRANCH}"^^; then
|
||||||
|
log_info "Main ref has no grandparent. No grand-deltas will be generated."
|
||||||
|
else
|
||||||
|
log_info "Generate static delta from main ref's grandparent"
|
||||||
|
if ! ostree --repo="${DEPLOY_REPO}" static-delta generate --from="${OSTREE_BRANCH}"^^ --to="${OSTREE_BRANCH}"; then
|
||||||
|
log_struc_error "Error generating static delta from main ref's grandparent."
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Update summary
|
||||||
|
log_struc_info "Update summary file" deploy_repo "${DEPLOY_REPO}"
|
||||||
|
if ! ostree --repo="${DEPLOY_REPO}" summary -u; then
|
||||||
|
log_struc_error "Error updating summary" status "$?"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
def compose_ostree():
|
# Deploy
|
||||||
print_log("Compose ostree")
|
log_struc_info "Deploy to web server" deploy_repo "${DEPLOY_REPO}" dest_repo "${DEST_REPO}"
|
||||||
time_fmt = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H%M%S")
|
if ! "$(pwd)/rsync-repos" --src "${DEPLOY_REPO}" --dest "${DEST_REPO}"; then
|
||||||
run_proc(
|
log_struc_error "Error deploying to web server" status "$?"
|
||||||
subprocess.list2cmdline(
|
exit
|
||||||
[
|
fi
|
||||||
"rpm-ostree",
|
|
||||||
"compose",
|
|
||||||
"tree",
|
|
||||||
"--unified-core",
|
|
||||||
f"--cachedir={CACHE_DIR}",
|
|
||||||
f"--repo={BUILD_REPO}",
|
|
||||||
f"--add-metadata-string=Build={time_fmt}",
|
|
||||||
WK_DIR.joinpath(args.get("--treefile")),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def sign_commit():
|
|
||||||
commit_id = subprocess.run(["ostree", f"--repo={DEPLOY_REPO}", "rev-parse", args.get("--ostree-branch")], capture_output=True, text=True)
|
|
||||||
|
|
||||||
with open(args.get("--gpg-passfile"), "r") as json_file:
|
|
||||||
gpg_data = json.loads(json_file.read())
|
|
||||||
|
|
||||||
print_log(f"Signing rpm-ostree commit {commit_id.stdout} with GPG key-id {gpg_data.get("gpg-id")}")
|
|
||||||
|
|
||||||
gpg_cmd = f"echo {gpg_data.get("passphrase")} | gpg --batch --always-trust --yes --passphrase-fd 0 --pinentry-mode=loopback -s $(mktemp)"
|
|
||||||
run_gpg_cmd = subprocess.Popen(gpg_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
||||||
output = run_gpg_cmd.communicate()[0]
|
|
||||||
if output:
|
|
||||||
print_log(f"Error unlocking GPG keyring: {output}")
|
|
||||||
exit(99)
|
|
||||||
|
|
||||||
run_proc(f"ostree --repo={DEPLOY_REPO} gpg-sign {commit_id.stdout} {gpg_data.get("gpg-id")}")
|
|
||||||
|
|
||||||
|
|
||||||
def prepare_deploy():
|
|
||||||
print_log("Prune refs older than 30 days")
|
|
||||||
run_proc(
|
|
||||||
f"ostree --repo={BUILD_REPO} prune --refs-only --keep-younger-than='30 days ago'"
|
|
||||||
)
|
|
||||||
|
|
||||||
print_log("Pull new ostree commit into DEPLOY_REPO")
|
|
||||||
run_proc(
|
|
||||||
f"ostree --repo={DEPLOY_REPO} pull-local --depth=1 {BUILD_REPO} {args.get("--ostree-branch")}"
|
|
||||||
)
|
|
||||||
|
|
||||||
print_log("Remove local build repo")
|
|
||||||
shutil.rmtree(BUILD_REPO, ignore_errors=True)
|
|
||||||
|
|
||||||
print_log("Check filesystem for errors")
|
|
||||||
run_proc(f"ostree --repo={DEPLOY_REPO} fsck {args.get("--ostree-branch")}")
|
|
||||||
|
|
||||||
|
|
||||||
def generate_deltas():
|
|
||||||
print_log("Check if main ref has parent")
|
|
||||||
check_parent = run_proc(
|
|
||||||
f"ostree --repo={DEPLOY_REPO} show {args.get("--ostree-branch")}", capture_output=True
|
|
||||||
)
|
|
||||||
|
|
||||||
if not check_parent.stderr:
|
|
||||||
print_log("Generate static delta from main ref's parent")
|
|
||||||
run_proc(
|
|
||||||
f"ostree --repo={DEPLOY_REPO} static-delta generate {args.get("--ostree-branch")}"
|
|
||||||
)
|
|
||||||
|
|
||||||
print_log("Check if main ref's parent has parent")
|
|
||||||
check_gparent = run_proc(
|
|
||||||
f"ostree --repo={DEPLOY_REPO} show {args.get("--ostree-branch")}^^",
|
|
||||||
capture_output=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not check_gparent.stderr:
|
|
||||||
print_log("Generate static delta from parent of main ref's parent")
|
|
||||||
run_proc(
|
|
||||||
f"ostree --repo={DEPLOY_REPO} static-delta generate --from={args.get("--ostree-branch")}^^ --to={args.get("--ostree-branch")}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
print_log("Main ref's parent has no parent. No deltas generated.")
|
|
||||||
else:
|
|
||||||
print_log("Main ref has no parent. No deltas generated.")
|
|
||||||
|
|
||||||
|
|
||||||
def update_summary():
|
|
||||||
print_log("Update summary file of DEPLOY_REPO")
|
|
||||||
run_proc(f"ostree --repo={DEPLOY_REPO} summary -u")
|
|
||||||
|
|
||||||
|
|
||||||
def deploy_repo():
|
|
||||||
print_log("Deploying repo to web server root")
|
|
||||||
if DEPLOY_REPO.exists():
|
|
||||||
run_proc(f"{BASE_DIR}/rsync-repos --src {DEPLOY_REPO} --dest {args.get("--dest-repo")}")
|
|
||||||
else:
|
|
||||||
print_log("DEPLOY_REPO not found. Not deploying to web server")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
args = docopt(__doc__, help=True, version="ostree-engine 0.1.0")
|
|
||||||
prepare_build_env()
|
|
||||||
compose_ostree()
|
|
||||||
prepare_deploy()
|
|
||||||
generate_deltas()
|
|
||||||
if args.get("--gpg-passfile"):
|
|
||||||
sign_commit()
|
|
||||||
update_summary()
|
|
||||||
if not args.get("--no-deploy"):
|
|
||||||
deploy_repo()
|
|
||||||
|
|||||||
9
src/borgmatic-2300.timer
Normal file
9
src/borgmatic-2300.timer
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Run borgmatic backup
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 23:00:00
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@ -22,16 +22,12 @@
|
|||||||
"copr:copr.fedorainfracloud.org:hyperreal:better_fonts"
|
"copr:copr.fedorainfracloud.org:hyperreal:better_fonts"
|
||||||
],
|
],
|
||||||
"add-files": [
|
"add-files": [
|
||||||
[
|
["borgmatic-config.yaml", "/etc/borgmatic/config.yaml"],
|
||||||
"borgmatic-config.yaml",
|
["user-dirs.defaults", "/etc/xdg/user-dirs.defaults"],
|
||||||
"/etc/borgmatic/config.yaml"
|
["borgmatic-2300.timer", "/etc/systemd/system/borgmatic-2300.timer"]
|
||||||
],
|
|
||||||
[
|
|
||||||
"user-dirs.defaults",
|
|
||||||
"/etc/xdg/user-dirs.defaults"
|
|
||||||
]
|
|
||||||
],
|
],
|
||||||
"units": [
|
"units": [
|
||||||
|
"borgmatic-2300.timer",
|
||||||
"snapper-cleanup.timer",
|
"snapper-cleanup.timer",
|
||||||
"snapper-timeline.timer"
|
"snapper-timeline.timer"
|
||||||
],
|
],
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user