jetpac/dotfiles
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Config cleanup, create_mrshughes refactoring

Browse Source
This commit is contained in:
Petr Nyc
2026-04-20 17:26:56 +02:00
parent 4bef91eb1b
commit 970dfb4c1e
12 changed files with 553 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files

419
bin/create_mrshughes
View File

@@ -1,197 +1,334 @@
#!/bin/zsh
set -e
set -x
set -eu
SCRIPT_NAME=${0:t}
SCRIPT_DIR=${0:A:h}
DEFAULT_PARENT_WS='ssh://pnyc@dabel.us.oracle.com//workspace/pnyc/solaris-reviews/on-sru'
DEFAULT_DEST_ROOT="${HOME}/PycharmProjects"
DEFAULT_FOLDER_PREFIX='PetrN/'
DEFAULT_POINT_OF_CONTACT='petr.nyc@oracle.com'
DEFAULT_SLACK_CHANNEL='@pnyc'
DEFAULT_PYTHON='/opt/homebrew/bin/python3.11'
OS_NAME="$(uname -s)"
export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin${PATH:+:$PATH}"
usage() {
print "Usage: ${0:t} [-r repo_url] [-d dest_dir] [-h]"
print ""
print " -r repo_url Mercurial repo URL to clone."
print " Default: ssh://pnyc@dabel.us.oracle.com//workspace/pnyc/solaris-reviews/on-sru"
print " -d dest_dir Local directory to clone into."
print " Default: ~/PycharmProjects/<repo_name>"
print " -h Show this help text."
print ""
print "The script exits if dest_dir already exists."
cat <<EOF
Usage: ${SCRIPT_NAME} [-r repo_url] [-d dest_dir] [-f folder_prefix] [-c point_of_contact] [-s slack_channel] [-p python_exe] [-j python_jenkins_path] [-h]
-r repo_url Mercurial repo URL to clone.
Default: ${DEFAULT_PARENT_WS}
-d dest_dir Local directory to clone into.
Default: ~/PycharmProjects/<repo_name>
-f folder_prefix Jenkins folder prefix.
Default: ${DEFAULT_FOLDER_PREFIX}
-c point_of_contact Contact email for generated config.
Default: ${DEFAULT_POINT_OF_CONTACT}
-s slack_channel Slack handle/channel for generated config.
Default: ${DEFAULT_SLACK_CHANNEL}
-p python_exe Python executable for virtualenv creation.
Default: ${DEFAULT_PYTHON}
-j python_jenkins_path Optional local checkout to inject into requirements.txt.
Default: disabled
-h Show this help text.
Environment overrides:
MRSHUGHES_PARENT_WS
MRSHUGHES_DEST_DIR
MRSHUGHES_FOLDER_PREFIX
MRSHUGHES_POINT_OF_CONTACT
MRSHUGHES_SLACK_CHANNEL
MRSHUGHES_PYTHON
MRSHUGHES_PYTHON_JENKINS_PATH
The script exits if dest_dir already exists.
EOF
}
die() {
print -u2 -- "$@"
exit 1
}
require_command() {
command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
}
resolve_executable() {
local executable=${~1}
if [[ "$executable" == */* ]]; then
[[ -x "$executable" ]] || die "Executable not found or not executable: $executable"
print -r -- "$executable"
return
fi
command -v "$executable" >/dev/null 2>&1 || die "Command not found in PATH: $executable"
command -v "$executable"
}
disable_proxy() {
local proxy_script="${SCRIPT_DIR}/proxy"
if [[ -f "$proxy_script" ]]; then
source "$proxy_script" off >/dev/null
fi
}
ensure_clone_identity_loaded() {
local ssh_host="$1"
local identity_file=""
local identity_pub=""
ssh_host="${ssh_host%%:*}"
local ssh_host=$1
local identity_file
local identity_pub
identity_file=$(ssh -G "$ssh_host" 2>/dev/null | awk '/^identityfile / {print $2; exit}')
if [[ -z "$identity_file" ]]; then
return 0
fi
[[ -n "$identity_file" ]] || return
identity_file=${~identity_file}
identity_pub="${identity_file}.pub"
if [[ ! -f "$identity_file" || ! -f "$identity_pub" ]]; then
return 0
fi
[[ -f "$identity_file" && -f "$identity_pub" ]] || return
if ssh-add -T "$identity_pub" >/dev/null 2>&1; then
return 0
return
fi
print "Loading SSH identity for ${ssh_host}: ${identity_file}"
ssh-add --apple-use-keychain "$identity_file" >/dev/null 2>&1 || ssh-add "$identity_file"
print -- "Loading SSH identity for ${ssh_host}: ${identity_file}"
if [[ "$OS_NAME" == "Darwin" ]]; then
ssh-add --apple-use-keychain "$identity_file" >/dev/null
else
ssh-add "$identity_file" >/dev/null
fi
}
DEFAULT_PARENT_WS='ssh://pnyc@dabel.us.oracle.com//workspace/pnyc/solaris-reviews/on-sru'
# Example repo URLs:
# ssh://pnyc@andel.us.oracle.com//workspace/pnyc/solaris-reviews/secure-integrate/userland11.4
# ssh://pnyc@andel.us.oracle.com//workspace/pnyc/solaris-reviews/userland-pipeline
# ssh://pnyc@andel.us.oracle.com//workspace/pnyc/solaris-reviews/akidr-text
# ssh://pnyc@andel.us.oracle.com//workspace/pnyc/solaris-reviews/akidr
PARENT_WS="$DEFAULT_PARENT_WS"
DEST_DIR=''
apply_python_jenkins_override() {
local checkout_path=$1
local tmp_requirements
while getopts ":r:d:h" opt; do
tmp_requirements=$(mktemp)
sed -E "s|^git.*$|git+file://${checkout_path}|" requirements.txt > "$tmp_requirements"
/bin/mv "$tmp_requirements" requirements.txt
}
yaml_escape() {
local value=$1
value=${value//\\/\\\\}
value=${value//\"/\\\"}
print -r -- "$value"
}
set_yaml_key() {
local file=$1
local key=$2
local value
local tmp_file
value=$(yaml_escape "$3")
tmp_file=$(mktemp)
if ! awk -v key="$key" -v value="$value" '
$0 ~ "^[[:space:]]*" key ":[[:space:]]*" {
match($0, /^[[:space:]]*/)
print substr($0, 1, RLENGTH) key ": \"" value "\""
changed = 1
next
}
{ print }
END { exit changed ? 0 : 1 }
' "$file" > "$tmp_file"; then
/bin/rm -f "$tmp_file"
die "Expected key not found in ${file}: ${key}"
fi
/bin/mv "$tmp_file" "$file"
}
insert_yaml_key_after() {
local file=$1
local anchor=$2
local key=$3
local value
local tmp_file
value=$(yaml_escape "$4")
tmp_file=$(mktemp)
if ! awk -v anchor="$anchor" -v key="$key" -v value="$value" '
{
print
if (!inserted && $0 ~ "^[[:space:]]*" anchor ":[[:space:]]*") {
match($0, /^[[:space:]]*/)
print substr($0, 1, RLENGTH) key ": \"" value "\""
inserted = 1
}
}
END { exit inserted ? 0 : 1 }
' "$file" > "$tmp_file"; then
/bin/rm -f "$tmp_file"
die "Expected anchor not found in ${file}: ${anchor}"
fi
/bin/mv "$tmp_file" "$file"
}
configure_defaults_devel() {
local file=$1
set_yaml_key "$file" script_dir_base "$SCRIPT_DIR_BASE"
if grep -Eq '^[[:space:]]*pipeline_workspace:[[:space:]]*' "$file"; then
set_yaml_key "$file" pipeline_workspace "$JENKINS_CLONE_FROM"
else
insert_yaml_key_after "$file" script_dir_base pipeline_workspace "$JENKINS_CLONE_FROM"
fi
set_yaml_key "$file" folder_prefix "$FOLDER_PREFIX"
set_yaml_key "$file" point_of_contact "$POINT_OF_CONTACT"
set_yaml_key "$file" slack_channel "$SLACK_CHANNEL"
}
find_lint_dir() {
if [[ -d "${DEST_DIR}/solaris/on/production" ]]; then
print -r -- "${DEST_DIR}/solaris/on/production"
return
fi
if [[ -d "${DEST_DIR}/solaris/userland/sru" ]]; then
print -r -- "${DEST_DIR}/solaris/userland/sru"
return
fi
die "Unable to determine lint directory under ${DEST_DIR}/solaris"
}
print_config() {
print -- "Parent workspace: ${PARENT_WS}"
print -- "Destination directory: ${DEST_DIR}"
print -- "Folder prefix: ${FOLDER_PREFIX}"
print -- "Point of contact: ${POINT_OF_CONTACT}"
print -- "Slack channel: ${SLACK_CHANNEL}"
print -- "Python executable: ${PYTHON_EXE}"
if [[ -n "$PYTHON_JENKINS_PATH" ]]; then
print -- "python-jenkins override: ${PYTHON_JENKINS_PATH}"
else
print -- "python-jenkins override: disabled"
fi
}
PARENT_WS="${MRSHUGHES_PARENT_WS:-$DEFAULT_PARENT_WS}"
DEST_DIR="${MRSHUGHES_DEST_DIR:-}"
FOLDER_PREFIX="${MRSHUGHES_FOLDER_PREFIX:-$DEFAULT_FOLDER_PREFIX}"
POINT_OF_CONTACT="${MRSHUGHES_POINT_OF_CONTACT:-$DEFAULT_POINT_OF_CONTACT}"
SLACK_CHANNEL="${MRSHUGHES_SLACK_CHANNEL:-$DEFAULT_SLACK_CHANNEL}"
PYTHON_EXE="${MRSHUGHES_PYTHON:-$DEFAULT_PYTHON}"
PYTHON_JENKINS_PATH="${MRSHUGHES_PYTHON_JENKINS_PATH:-}"
while getopts ":r:d:f:c:s:p:j:h" opt; do
case "$opt" in
r)
PARENT_WS="$OPTARG"
;;
d)
DEST_DIR="$OPTARG"
;;
r) PARENT_WS="$OPTARG" ;;
d) DEST_DIR="$OPTARG" ;;
f) FOLDER_PREFIX="$OPTARG" ;;
c) POINT_OF_CONTACT="$OPTARG" ;;
s) SLACK_CHANNEL="$OPTARG" ;;
p) PYTHON_EXE="$OPTARG" ;;
j) PYTHON_JENKINS_PATH="$OPTARG" ;;
h)
usage
exit 0
;;
:)
print -u2 "Missing argument for -$OPTARG"
usage >&2
exit 1
die "Missing argument for -$OPTARG"
;;
\?)
print -u2 "Unknown option: -$OPTARG"
usage >&2
exit 1
die "Unknown option: -$OPTARG"
;;
esac
done
shift $((OPTIND - 1))
[[ $# -eq 0 ]] || die "Unexpected positional arguments: $*"
if [[ $# -ne 0 ]]; then
print -u2 "Unexpected positional arguments: $*"
usage >&2
exit 1
fi
for command_name in hg awk sed mktemp grep make; do
require_command "$command_name"
done
# remove trailing slash
PARENT_WS="${PARENT_WS%/}"
REPO=${PARENT_WS##*/}
[[ -n "$DEST_DIR" ]] || DEST_DIR="${DEFAULT_DEST_ROOT}/${REPO}"
REPO=${PARENT_WS##*/} # userland11.4
DEST_DIR=${~DEST_DIR:A}
PYTHON_EXE=$(resolve_executable "$PYTHON_EXE")
if [[ -z "$DEST_DIR" ]]; then
DEST_DIR=~/PycharmProjects/${REPO}
if [[ -n "$PYTHON_JENKINS_PATH" ]]; then
PYTHON_JENKINS_PATH=${~PYTHON_JENKINS_PATH:A}
[[ -d "$PYTHON_JENKINS_PATH" ]] || die "Required directory not found: ${PYTHON_JENKINS_PATH}"
fi
DEST_DIR=${~DEST_DIR}
DEST_DIR=${DEST_DIR:A}
[[ ! -e "$DEST_DIR" ]] || die "Destination already exists: ${DEST_DIR}"
if [[ -e "$DEST_DIR" ]]; then
print -u2 "Destination already exists: $DEST_DIR"
exit 1
fi
HG_CLONE_ARGS=()
JENKINS_CLONE_FROM=$PARENT_WS
if [[ "$PARENT_WS" == ssh://* ]]; then
SSH_CLONE_HOST=${${PARENT_WS#ssh://}%%/*}
SSH_CLONE_HOST=${SSH_CLONE_HOST#*@}
ensure_clone_identity_loaded "$SSH_CLONE_HOST"
# Avoid exhausting ssh-agent identities before ssh reaches the host-specific
# IdentityFile from ~/.ssh/config (for example ~/.ssh/dabel.key).
HG_CLONE_ARGS+=(--config "ui.ssh=ssh -o BatchMode=yes -o IdentitiesOnly=yes")
JENKINS_CLONE_FROM="ssh://${${PARENT_WS#ssh://}#*@}"
ensure_clone_identity_loaded "${${PARENT_WS#ssh://}%%/*}"
fi
JENKINS_CLONE_FROM="ssh://${${PARENT_WS#ssh://}#*@}"
FOLDER_PREFIX='PetrN/'
POINT_OF_CONTACT='petr.nyc@oracle.com'
SLACK_CHANNEL='@pnyc'
SCRIPT_DIR_BASE=$PARENT_WS
if [[ "$PARENT_WS" == *'//'* ]]; then
SCRIPT_DIR_BASE="/${PARENT_WS##*//}"
fi
# /workspace/pnyc/solaris-reviews/secure-integrate/userland11.4
SCRIPT_DIR_BASE=$(echo "$PARENT_WS" | awk '{sub(/^.*\/\//,"/"); print}')
MV=/bin/mv
RM=/bin/rm
CP=/bin/cp
CAT=/bin/cat
print_config
pwd
print -- "Validating Mercurial access to ${PARENT_WS}"
hg identify "$PARENT_WS" >/dev/null
print -- "Cloning ${PARENT_WS} into ${DEST_DIR}"
mkdir -p "${DEST_DIR:h}"
hg "${HG_CLONE_ARGS[@]}" clone "$PARENT_WS" "$DEST_DIR"
pwd
hg clone "$PARENT_WS" "$DEST_DIR"
cd "$DEST_DIR"
pwd
source proxy off
$RM -rf venv
for required_path in \
requirements.txt \
Makefile.inc \
common/tools/create_virtualenv \
common/etc/passwd.template \
common/jobs/defaults.devel.tmpl \
common/jobs/defaults.stage.tmpl
do
[[ -e "$required_path" ]] || die "Required path not found: ${DEST_DIR}/${required_path}"
done
# this holds upgraded python-jenkins - will it work?
sed -E 's/^git.*$/git+file:\/\/\/Users\/jetpac\/PycharmProjects\/python-jenkins/' requirements.txt > /tmp/r
mv /tmp/r requirements.txt
[[ -x common/tools/create_virtualenv ]] || die "Required executable not found: ${DEST_DIR}/common/tools/create_virtualenv"
common/tools/create_virtualenv /opt/homebrew/bin/python3.11 requirements.txt venv
disable_proxy
/bin/rm -rf venv
echo '[alias]' >> .hg/hgrc
echo 'ci = ci -X Makefile.inc' >> .hg/hgrc
echo 'st = st -X Makefile.inc' >> .hg/hgrc
tmpmake=$(mktemp)
sed 's:PYTHON3=python3.7:PYTHON3=python3.11:g' < Makefile.inc > "$tmpmake"
$MV "$tmpmake" Makefile.inc
# set up pwd
cd "$DEST_DIR/common/etc"
$CP passwd.template passwd
# set up dev defaults
cd "$DEST_DIR/common/jobs/"
cp defaults.devel.tmpl defaults.devel.yml
cp defaults.stage.tmpl defaults.stage.yml
$CAT > defaults_devel_patch <<- CATT
9c9
< script_dir_base: "/workspace/pzahradn/jenkins/mrspatmore"
---
> script_dir_base: "${SCRIPT_DIR_BASE=}"
11a12
> pipeline_workspace: "${JENKINS_CLONE_FROM}"
14c15
< folder_prefix: "pez-" # Could be used to deploy the devel jobs to different jenkins folder
---
> folder_prefix: "${FOLDER_PREFIX}" # Could be used to deploy the devel jobs to different jenkins folder
16c17
< point_of_contact: "petr.zahradnik@oracle.com"
---
> point_of_contact: "${POINT_OF_CONTACT}"
21c22
< slack_channel: "@pzahradn"
---
> slack_channel: "${SLACK_CHANNEL}"
CATT
patch -p0 defaults.devel.yml < defaults_devel_patch
if [[ -d "$DEST_DIR/solaris/on/production" ]]; then
LINT_DIR="$DEST_DIR/solaris/on/production"
elif [[ -d "$DEST_DIR/solaris/userland/sru" ]]; then
LINT_DIR="$DEST_DIR/solaris/userland/sru"
else
print -u2 "Unable to determine lint directory under $DEST_DIR/solaris"
exit 1
if [[ -n "$PYTHON_JENKINS_PATH" ]]; then
print -- "Applying local python-jenkins override from ${PYTHON_JENKINS_PATH}"
apply_python_jenkins_override "$PYTHON_JENKINS_PATH"
fi
source proxy off
/usr/bin/env PATH="$PATH" /bin/bash -lc 'common/tools/create_virtualenv "$@"' bash "$PYTHON_EXE" requirements.txt venv
{
print -- '[alias]'
print -- 'ci = ci -X Makefile.inc'
print -- 'st = st -X Makefile.inc'
} >> .hg/hgrc
tmp_makefile=$(mktemp)
sed 's:PYTHON3=python3.7:PYTHON3=python3.11:g' Makefile.inc > "$tmp_makefile"
/bin/mv "$tmp_makefile" Makefile.inc
/bin/cp common/etc/passwd.template common/etc/passwd
/bin/cp common/jobs/defaults.devel.tmpl common/jobs/defaults.devel.yml
/bin/cp common/jobs/defaults.stage.tmpl common/jobs/defaults.stage.yml
configure_defaults_devel common/jobs/defaults.devel.yml
LINT_DIR=$(find_lint_dir)
disable_proxy
cd "$LINT_DIR"
make FAKE_DEVEL_ENV=yes lint