add CI harness and worker

These versions are slightly ahead of what I have elsewhere in an "emacs-local-ci" project. 

I modified them to:
 - allow running multiple copies of `crude-ci` at the same time
 - make outputs and intermediary artifacts under UCRT64 unique
This commit is contained in:
Corwin Brust 2026-05-12 19:22:04 -05:00
parent 386c678bd9
commit 535a516d4f
2 changed files with 502 additions and 0 deletions

360
build-dist.sh Normal file
View file

@ -0,0 +1,360 @@
#!/usr/bin/bash
# build-dist - build and package Emacs for Windows
#
# Copyright 2023 Corwin Brust <corwin@bru.st>
#
# This program is distributed under the terms of the GNU Public
# License version 3 or (at your option) any later version.
#
# E.g.:
#(cd /g; for v in 29 30 ; do ( MV=$v; ( cd git/emacs-$v; git pull --rebase) ; MV=$v ./build-dist.sh) |tee log/emacs-$MV-`date +%Y-%m-%d-%H%M`.log ; done )
function url_encode {
(echo "$1" | jq -Rr @uri) 2>/dev/null
}
function say {
local rev=$1
local branch=$2
local who=$( url_encode "$3")
local where=$4
local what=$5
local site="http://comic.chat:8998"
local u1="$site/$who/%23${where}/$(url_encode "[$rev] $what")"
local u2="$site/$who/%23emacs-dev/$(url_encode "[$branch@$rev] $what")"
local url
local RC
for url in $u1 $u2 ; do
curl "$url" >/dev/null 2>&1 ;
done
# http://ws.comic.chat:8998/eliz/%23emacs-29/8c1b10
RC=$?
}
# most likely things to edit
TO=${TO:-/h}
FROM=${FROM:-/g}
MV=${MV:-31}
EC=${EC:-"$( echo "$MSYSTEM" | grep UCRT64 )"}
EU=${EU:-"$(test -n "$EC" && printf '%s' "-ucrt")"}
SRC=${SRC:-$FROM/git/emacs${EU}-${MV}}
DIR=${DIR:-$TO/template.directive}
#SOURCE_BMP=/g/emacs.bmp
SOURCE_BMP=$SRC/etc/images/splash.bmp
SOURCE_NSI=/g/emacs.nsi
#SOURCE_NSI=$SRC/admin/nt/dist-build/emacs.nsi
SOURCE_COPYING=$SRC/COPYING
# get current the version for the main development branch
SVH_CGIT="http://cgit.git.savannah.gnu.org/cgit/emacs.git/plain/configure.ac"
MASTER_VERSION=$(wget -qO - $SVH_CGIT \
| grep AC_INIT \
| perl -ne 'print $1 if /(\d+\.\d+(?:\.\d+)?)/');
# get the first component of master's version, for later
MASTER_VERSION_MAJOR_VERSION=$(echo $MASTER_VERSION | cut -d . -f 1)
# and the local version
EV=${EV-$(grep '^AC_INIT' $SRC/configure.ac 2>/dev/null |perl -ne 'print $1 if /(\d+\.\d+\.\d+)/')}
DEPS=${DEPS:-$FROM/deps/emacs-${MV}${EU}-deps.zip}
SHORT_VER=${SHORT_VER:-$(cd $SRC; git rev-parse --short=6 HEAD)}
LONG_VER=${LONG_VER:-$(cd $SRC; git rev-parse HEAD)}
SLUG=${SLUG:-${MV}${EU}-$SHORT_VER}
EB=${EB:-"emacs-${SLUG}"}
EL=${EL:-"emacs-${EV}-$SHORT_VER"}
IB=${IB:-"${TO}/install"}
IN=${IN:-"${IB}/${EB}"}
UP=${UP:-"${FROM}/upload/${EB}"}
NO=${NO:-"${UP}/${EL}-no-deps.zip"}
FU=${FU:-"${UP}/${EL}.zip"}
SZ=${SZ:-"${UP}/${EL}-src.zip"}
SE=${SE:-"${UP}/${EL}-installer.exe"}
OE="${IB}/emacs-${EV}-installer.exe"
SXS=$IB/emacs-${EV:-"${MV}.0.50"}
# we are building when the source branch the same version as master
if [[ $MV -eq $MASTER_VERSION_MAJOR_VERSION ]] ; then
BRANCH=master
else
BRANCH=emacs-$MV
fi
CC_BRANCH=$( cd $SRC; git rev-parse --abbrev-ref HEAD )
CC_WHO="$(hostname)"
CC_WHERE="emacs-$MV"
echo "SVH: ${SVH_CGIT}
SV: ${MASTER_VERSION}
MM: ${MASTER_VERSION_MAJOR_VERSION}
MV: ${MV}
BR: ${BRANCH}
EC: ${EC}
EU: ${EU}
TO: ${TO}
FROM: ${FROM}
SRC: ${SRC}
DEPS: ${DEPS}
SREV: ${SHORT_VER}
LREV: ${LONG_VER}
SLUG: ${SLUG}
EB: ${EB}
IB: ${IB}
IN: ${IN}
UP: ${UP}
NO: ${NO}
FU: ${FU}
SZ: ${SZ}
SE: ${SE}
OE: ${OE}
DIR: ${DIR}
EV: ${EV}
SXS: ${SXS}
WHO: $CC_WHO
WHER: $CC_WHERE
BRNC: $CC_BRANCH"
if [[ -z "$SHORT_VER" ]] ; then
echo "Failed to extract git revision (short)"
exit 1;
fi
if [[ -z "$LONG_VER" ]] ; then
echo "Failed to extract git revision"
exit 1;
fi
sleep 10;
[ -r ${FROM}/.private ] && . ${FROM}/.private;
if [[ -d $IN ]] ; then
echo "NOTICE: build dir exits: $IN"
else
CC_WHAT="Windows build started"
#say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT";
echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
# (cd $SRC; git clean -fxd;
# ((./autogen.sh \
# && ./configure --with-modules \
# --without-dbus \
# --with-native-compilation=aot \
# --without-compress-install \
# --with-tree-sitter \
# CFLAGS='-O2' \
# && make install V=1 -j 10101010101010101010 \
# prefix=$IN ) 2>&1 | tee $TO/log/${EB}-make.log
# ) && ( echo "1..OK make" \
# ; (printf '%s' "$SHORT_VER" >/g/emacs-$MV.git-revision ) \
# ; true)
# ) || (echo "ERROR: prep upload ($?)"; exit 1);
# (cd $SRC; git clean -fxd 2>&1 >/dev/null;
# ((./autogen.sh \
# && make install V=1 -j10 \
# configure="--disable-acl --with-modules --without-dbus --with-native-compilation=aot --without-compress-install --with-tree-sitter CFLAGS='-O2'" \
# prefix=$IN ) 2>&1 | tee $TO/log/${EB}-make.log
# ) && ( echo "1..OK make" \
# ; (printf '%s' "$SHORT_VER" >/g/emacs-$MV.git-revision ) \
# ; true)
# ) || (echo "ERROR: prep upload ($?)"; exit 1);
(cd $SRC; git clean -fxd 2>&1 >/dev/null;
((make install V=1 -j10 \
configure="--prefix=$IN --disable-acl --with-modules --without-dbus --with-native-compilation=aot --without-compress-install --with-tree-sitter CFLAGS='-O0 -g3'" \
) 2>&1 | tee $TO/log/${EB}-make.log
) && ( echo "1..OK make" \
; (printf '%s' "$SHORT_VER" >/g/emacs-$MV.git-revision ) \
; true)
) || (echo "ERROR: prep upload ($?)"; exit 1)
fi
sleep 17
if [[ ! -d $IN ]]; then
echo "ERROR install folder is missing: $IN"
exit 1
fi
if [[ -r $NO ]]; then
echo "NOTICE: no-deps zip exits: $NO"
else
CC_WHAT="Windows: creating no-deps.zip"
#say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT";
echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
((mkdir -p $UP \
&& cd $IN \
&& zip -vr9 $NO . 2>&1) 2>&1 >$TO/log/${EB}-zip-deps.log \
&& echo "2..OK zip nodeps") \
|| ( echo "FAILED ($?)" ; exit 2 )
fi
if [[ -r $FU ]]; then
echo "NOTICE: full zip exists: $FU"
else
CC_WHAT="Windows creating full.zip"
#say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT";
echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
((cd $IN \
&& unzip -d bin $DEPS 2>&1) 2>&1 >$TO/log/${EB}-unzip-deps.log \
&& echo "3..OK unzip deps") \
|| ( echo "FAILED ($?)" ; exit 3 )
((cd $IN \
&& zip -vr9 $FU .) 2>&1 >$TO/log/${EB}-zip.log \
&& echo "4..OK rezip full") \
|| ( echo "FAILED ($?)" ; exit 4 )
fi
MAKE_SE=1;
if [[ -r $SE ]] ; then
echo "NOTICE: self-installer exists: $SE";
if [[ -z $FORCE_SE ]] ; then
# noop?
MAKE_SE=0;
else
echo "FORCE: building SE anyway (FOCE_SE=$FORCE_SE)"
fi
fi
if [[ $MAKE_SE ]]; then
#sleep 10;
if [[ -d $SXS ]] ; then
echo "WARNING: Self-install source dir found, reusing: $SXS";
sleep 90;
else
mv $IN $SXS ||
( echo "ERROR: installer source mv failed ($?): $IN => $SXS";
exit 5; )
fi
# DIST_SV=$IB/emacs-$EV
# CC_WHAT="Windows: moving installed Emacs to $DIST_SV"
# echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
# sleep 2;
# #rm -rf $DIST_SV;
# mv $IN $DIST_SV;
CC_WHAT="Windows: creating self-installer"
#say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT";
echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
((cd $IB \
&& cp $SOURCE_BMP . \
&& cp $SOURCE_NSI . \
&& cp $SOURCE_COPYING emacs-$EV \
&& makensis -v4 \
-DEMACS_VERSION=$EV \
-DVERSION_BRANCH=$EV \
-DOUT_VERSION=$EV \
emacs.nsi \
&& mv $OE $SE && mv $IB/$EB $IN) \
| tee $TO/log/${EB}-esi.log \
&& echo "5..OK executable self installer") \
|| (echo "ERROR: creating self installer ($?)"; exit 5)
# CC_WHAT="Windows: putting installed Emacs back in $IN"
# echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
# sleep 2;
# mv $DIST_SV $IN;
# archive self-installer sources
if [[ -d $IN ]] ; then
echo "NOTICE: Found self-installer source archive, leaving: $SXS";
else
mv $SXS $IN ||
( echo "ERROR: source restore mv failed ($?): $IN => $SXS";
exit 5; )
fi
fi
# archive sources
if [[ -r $SZ ]] ; then
echo "NOTICE: source zip exists: $SZ";
else
CC_WHAT="Windows: creating src archive"
#say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT";
echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
# clean-up the build folder
cd $SRC
git clean -fxd 2>&1 >$TO/log/${EB}-clea.log
# archive sources, omit git cruft
((zip -9r $SZ . -x .git/ .git/\* 2>&1 >$TO/log/${EB}-src.log \
) && echo "6..OK archive sources"
) || (echo "ERROR: archive sources ($?)"; exit 4);
fi
cd $UP
# create SHA256 sums
if [[ $( ls -t *.{exe,zip,txt} 2>/dev/null | head -1 ) \
!= \
$( ls *.txt 2>/dev/null ) ]] ;
then
CC_WHAT="Windows: writing SHA256 sums"
#say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT";
echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
((for f in *.{zip,exe} ;
do
sha256sum.exe $f ;
done) | tee $UP/${EL}-sha256sums.txt
) | tee $TO/log/${EB}-sums.log
fi
# sign release files
EXTS=exe,zip,txt
if [[ $( ls $UP/*.{$EXTS} 2>/dev/null | wc -l ) \
-ne \
$( ls $UP/*.sig 2>/dev/null | wc -l) ]] ;
then
CC_WHAT="Windows: signing files"
#say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT";
echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"
(for f in $UP/*.{txt,exe,zip} ;
do
gpg --pinentry-mode=loopback \
--passphrase-file=$GPG_PPF \
--batch --yes -b $f
done) | tee $TO/lop/emacs-${SLUG}-sign.log
exit 0
fi
# create upload directives
if [[ $( ls $UP/*.{$EXTS} 2>/dev/null | wc -l ) \
-ne \
$( ls $UP/*.directive.asc 2>/dev/null | wc -l) ]] ;
then
(for f in *.{zip,exe,txt} ;
do
cat $DIR \
| perl -p \
-e "s/__FILE__/$f/msg;" \
-e "s/__MAJOR_VERSION__/$MV/msg;" \
-e "s/__VERSION__/${EV:=$MV.0.50}/msg;" \
> $f.directive ;
done) | tee $TO/log/emacs-${SLUG}-dirs.log
# sign directives
(((for f in *.directive ;
do
gpg --pinentry-mode=loopback \
--passphrase-file=$HOME/emacs-build/foo.txt \
--batch --yes --clearsign $f ;
done) | tee $TO/log/emacs-${SLUG}-sidr.log
) && echo "7..OK prep upload"
) || (echo "ERROR: prep upload ($?)"; exit 4);
fi
rsync -vvrte "/usr/bin/ssh -i $SSH_KEY" "$UP" "${SSH_USER}@corwin.bru.st:~/corwin-emacs/emacs${EU}-$MV"
ssh -i $SSH_KEY ${SSH_USER}@corwin.bru.st 'cd ~/corwin-emacs/emacs-32; ./update-sym-links.sh'
CC_WHAT="Windows build complete"
CC_WHAT="$CC_WHAT https://corwin.bru.st/emacs-$MV/emacs-$MV-$SHORT_VER"
say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT";
echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT"

142
crude-ci.sh Normal file
View file

@ -0,0 +1,142 @@
#!/usr/bin/bash
# crude-ci - build new Emacs binaries for windows after commits6
#
# Copyright 2026 Corwin Brust <corwin@bru.st>
#
# This program is distributed under the terms of the GNU Public
# License version 3 or (at your option) any later version.
#
test -n "$DEBUG" && set -x
# most likely things to edit
SLEEP_TIME=${SLEEP_TIME-17}
# if truthy, try to rebuild tree-sitter grammers also
# NO_TS=${NO_TS-1}
# number of loops between tree sitter project checks
TS_WAIT=$((17 * 60))
LK_WAIT=$((17 * 60))
EC=${EC:-"$( echo "$MSYSTEM" | grep UCRT64 )"}
EU=${EU:-"$(test -n "$EC" && printf '%s' "-ucrt")"}
# the name the current master branch will have when it becomes the release branch
MASTER=emacs$EU-32
# webpage with the head-revision of a given branch
BASE_URI=$( printf '%s://%s/%s?project=%s&property=%s&' \
'https' \
'test.comic.chat' \
'last-rev.pl' \
'emacs' \
'revision' )
# counter
runs_since_last_ts=$TS_WAIT
LOCKFILE=/g/.crude-ci.lock
if test -r $LOCKFILE ; then
echo "ERROR: cannot start while lockfile exists, try:\n\trm $LOCKFILE";
exit 1;
fi
# the .private file exports SSH_KEY and SSH_USER, needed for rsync
if [[ -r /g/.private ]] ; then
. /g/.private
else
echo "Private settings missing";
exit 1;
fi
sleep 10;
while true ;
do
if test -r $LOCKFILE ; then
echo -e "\e[1;6m$( date +'%Y-%m-%d %H:%M' ) Build in process. Trying again in $LK_WAIT seconds..\e[0m"
sleep $LK_WAIT
continue
fi
# maybe rebuild tree-sitter grammers
# NOTE: only build TS grammers under MSYS54 for now
if test -z "$NO_TS" && test -z "$EC" ; then
runs_since_last_ts=$((1+ runs_since_last_ts))
if [[ $runs_since_last_ts > $TS_WAIT ]] ; then
touch $LOCKFILE
echo -e "\e[1;2m$( date +'%Y-%m-%d %H:%M' ) Checking treesitter projects...\e[0m"
runs_since_last_ts=$((0));
/g/emacs-local-ci/build-tree-sitter.sh
rm $LOCKFILE
fi
fi
for VER in 32 31 ;
do
branch_folder_name=emacs$EU-$VER
cd /g/git/$branch_folder_name
# special case name of the master branch
branch_name=emacs-$VER #$branch_folder_name
if [[ $MASTER -eq $branch_folder_name ]] ; then
branch_name=master
branch_color=35
else
branch_color=36
fi
# echo "checking Emacs branche $branch_name for changes..."
local_rev=$( git rev-parse HEAD )
echo "Fetching... ${BASE_URI}&branch=${branch_name}"
remote_rev=$( wget -qO - $BASE_URI'&'branch=$branch_name );
if [[ "$local_rev" != "$remote_rev" ]] ; then
local_color=33
else
local_color=32
fi
echo -e "\e[1;2m$( date +'%Y-%m-%d %H:%M' )\e[0m " \
"\e[1;${branch_color}m$branch_folder_name\e[0m " \
"\e[1;2mhave:\e[0m\e[1;32m${local_rev:0:6}\e[0m " \
"\e[1;2mwant:\e[0m\e[1;${local_color}m${remote_rev:0:6}\e[0m"
if [[ -n $FORCE ]] || [[ "$local_rev" != "$remote_rev" ]] ; then
touch $LOCKFILE
# wait, in case of slow trickle of related commits
if [[ -z $FORCE ]] ; then
echo "pausing for $SLEEP_TIME seconds..";
sleep $SLEEP_TIME;
fi
echo "GIT: pull --rebase .."
git pull --rebase;
new_rev=$( git rev-parse HEAD );
new_rev_short=${new_rev:0:6};
(set -x;
MV=$VER /g/emacs-local-ci/build-dist.sh;
)2>&1 | tee -a /g/log/${branch_folder_name}-$new_rev_short-`date +%Y-%m-%d-%H%M`.log
problems=`(/g/emacs-local-ci/check-feature.pl ${branch_folder_name}-$new_rev_short; /g/emacs-local-ci/check-log.sh ${branch_folder_name}) | grep NOT | cut -d ' ' -f 1,4 | perl -e 'END{ print join q(, ), @a }' -ne 's/\s+/ /msg; push @a, $_ if length'`;
if [[ -n $problems ]] ; then
health="dubious: $problems";
else
health="OK"
fi
if [[ -n $CC_SSH_KEY ]] && [[ -n $CC_SSH_USER ]] && [[ -n $CC_SSH_HOST ]] ; then
#last_up="$( basename $( ls -1td /h/upload/$branch_folder_name* | head -1 ) )"
BUILD_URI="https://corwin.bru.st/$branch_folder_name/$branch_folder_name-$new_rev_short"
ssh -i "$CC_SSH_KEY" "$CC_SSH_USER@$CC_SSH_HOST" \
"echo '#emacs-dev [$branch_name@$new_rev_short] `uname -o` build $health $BUILD_URI (`hostname`)' >>/petroglyph/udp2irc/gliphy.fifo"
fi
rm $LOCKFILE
fi
done
FORCE=
powershell /g/emacs-local-ci/fix-shortcut.ps1
sleep ${SLEEP_TIME}
done