diff --git a/build-dist.sh b/build-dist.sh new file mode 100644 index 0000000..ea99782 --- /dev/null +++ b/build-dist.sh @@ -0,0 +1,360 @@ +#!/usr/bin/bash +# build-dist - build and package Emacs for Windows +# +# Copyright 2023 Corwin Brust +# +# This program is distributed under the terms of the GNU Public +# License version 3 or (at your option) any later version. +# + +# E.g.: +#(cd /g; for v in 29 30 ; do ( MV=$v; ( cd git/emacs-$v; git pull --rebase) ; MV=$v ./build-dist.sh) |tee log/emacs-$MV-`date +%Y-%m-%d-%H%M`.log ; done ) + +function url_encode { + (echo "$1" | jq -Rr @uri) 2>/dev/null +} + +function say { + local rev=$1 + local branch=$2 + local who=$( url_encode "$3") + local where=$4 + local what=$5 + local site="http://comic.chat:8998" + local u1="$site/$who/%23${where}/$(url_encode "[$rev] $what")" + local u2="$site/$who/%23emacs-dev/$(url_encode "[$branch@$rev] $what")" + local url + local RC + for url in $u1 $u2 ; do + curl "$url" >/dev/null 2>&1 ; + done + # http://ws.comic.chat:8998/eliz/%23emacs-29/8c1b10 + RC=$? +} + + +# most likely things to edit +TO=${TO:-/h} +FROM=${FROM:-/g} +MV=${MV:-31} +EC=${EC:-"$( echo "$MSYSTEM" | grep UCRT64 )"} +EU=${EU:-"$(test -n "$EC" && printf '%s' "-ucrt")"} +SRC=${SRC:-$FROM/git/emacs${EU}-${MV}} +DIR=${DIR:-$TO/template.directive} + +#SOURCE_BMP=/g/emacs.bmp +SOURCE_BMP=$SRC/etc/images/splash.bmp + +SOURCE_NSI=/g/emacs.nsi +#SOURCE_NSI=$SRC/admin/nt/dist-build/emacs.nsi + +SOURCE_COPYING=$SRC/COPYING + + +# get current the version for the main development branch +SVH_CGIT="http://cgit.git.savannah.gnu.org/cgit/emacs.git/plain/configure.ac" +MASTER_VERSION=$(wget -qO - $SVH_CGIT \ + | grep AC_INIT \ + | perl -ne 'print $1 if /(\d+\.\d+(?:\.\d+)?)/'); +# get the first component of master's version, for later +MASTER_VERSION_MAJOR_VERSION=$(echo $MASTER_VERSION | cut -d . -f 1) + +# and the local version +EV=${EV-$(grep '^AC_INIT' $SRC/configure.ac 2>/dev/null |perl -ne 'print $1 if /(\d+\.\d+\.\d+)/')} + +DEPS=${DEPS:-$FROM/deps/emacs-${MV}${EU}-deps.zip} +SHORT_VER=${SHORT_VER:-$(cd $SRC; git rev-parse --short=6 HEAD)} +LONG_VER=${LONG_VER:-$(cd $SRC; git rev-parse HEAD)} +SLUG=${SLUG:-${MV}${EU}-$SHORT_VER} + +EB=${EB:-"emacs-${SLUG}"} +EL=${EL:-"emacs-${EV}-$SHORT_VER"} +IB=${IB:-"${TO}/install"} +IN=${IN:-"${IB}/${EB}"} +UP=${UP:-"${FROM}/upload/${EB}"} +NO=${NO:-"${UP}/${EL}-no-deps.zip"} +FU=${FU:-"${UP}/${EL}.zip"} +SZ=${SZ:-"${UP}/${EL}-src.zip"} +SE=${SE:-"${UP}/${EL}-installer.exe"} + +OE="${IB}/emacs-${EV}-installer.exe" +SXS=$IB/emacs-${EV:-"${MV}.0.50"} + +# we are building when the source branch the same version as master +if [[ $MV -eq $MASTER_VERSION_MAJOR_VERSION ]] ; then + BRANCH=master +else + BRANCH=emacs-$MV +fi + +CC_BRANCH=$( cd $SRC; git rev-parse --abbrev-ref HEAD ) +CC_WHO="$(hostname)" +CC_WHERE="emacs-$MV" + +echo "SVH: ${SVH_CGIT} + SV: ${MASTER_VERSION} + MM: ${MASTER_VERSION_MAJOR_VERSION} + MV: ${MV} + BR: ${BRANCH} + EC: ${EC} + EU: ${EU} + TO: ${TO} +FROM: ${FROM} + SRC: ${SRC} +DEPS: ${DEPS} +SREV: ${SHORT_VER} +LREV: ${LONG_VER} +SLUG: ${SLUG} + EB: ${EB} + IB: ${IB} + IN: ${IN} + UP: ${UP} + NO: ${NO} + FU: ${FU} + SZ: ${SZ} + SE: ${SE} + OE: ${OE} + DIR: ${DIR} + EV: ${EV} + SXS: ${SXS} + WHO: $CC_WHO +WHER: $CC_WHERE +BRNC: $CC_BRANCH" + +if [[ -z "$SHORT_VER" ]] ; then + echo "Failed to extract git revision (short)" + exit 1; +fi + +if [[ -z "$LONG_VER" ]] ; then + echo "Failed to extract git revision" + exit 1; +fi + +sleep 10; + +[ -r ${FROM}/.private ] && . ${FROM}/.private; + +if [[ -d $IN ]] ; then + echo "NOTICE: build dir exits: $IN" +else + CC_WHAT="Windows build started" + #say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT"; + echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + + # (cd $SRC; git clean -fxd; + # ((./autogen.sh \ + # && ./configure --with-modules \ + # --without-dbus \ + # --with-native-compilation=aot \ + # --without-compress-install \ + # --with-tree-sitter \ + # CFLAGS='-O2' \ + # && make install V=1 -j 10101010101010101010 \ + # prefix=$IN ) 2>&1 | tee $TO/log/${EB}-make.log + # ) && ( echo "1..OK make" \ + # ; (printf '%s' "$SHORT_VER" >/g/emacs-$MV.git-revision ) \ + # ; true) + # ) || (echo "ERROR: prep upload ($?)"; exit 1); + # (cd $SRC; git clean -fxd 2>&1 >/dev/null; + # ((./autogen.sh \ + # && make install V=1 -j10 \ + # configure="--disable-acl --with-modules --without-dbus --with-native-compilation=aot --without-compress-install --with-tree-sitter CFLAGS='-O2'" \ + # prefix=$IN ) 2>&1 | tee $TO/log/${EB}-make.log + # ) && ( echo "1..OK make" \ + # ; (printf '%s' "$SHORT_VER" >/g/emacs-$MV.git-revision ) \ + # ; true) + # ) || (echo "ERROR: prep upload ($?)"; exit 1); + + (cd $SRC; git clean -fxd 2>&1 >/dev/null; + ((make install V=1 -j10 \ + configure="--prefix=$IN --disable-acl --with-modules --without-dbus --with-native-compilation=aot --without-compress-install --with-tree-sitter CFLAGS='-O0 -g3'" \ + ) 2>&1 | tee $TO/log/${EB}-make.log + ) && ( echo "1..OK make" \ + ; (printf '%s' "$SHORT_VER" >/g/emacs-$MV.git-revision ) \ + ; true) + ) || (echo "ERROR: prep upload ($?)"; exit 1) +fi + +sleep 17 + +if [[ ! -d $IN ]]; then + echo "ERROR install folder is missing: $IN" + exit 1 +fi + +if [[ -r $NO ]]; then + echo "NOTICE: no-deps zip exits: $NO" +else + CC_WHAT="Windows: creating no-deps.zip" + #say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT"; + echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + ((mkdir -p $UP \ + && cd $IN \ + && zip -vr9 $NO . 2>&1) 2>&1 >$TO/log/${EB}-zip-deps.log \ + && echo "2..OK zip nodeps") \ + || ( echo "FAILED ($?)" ; exit 2 ) +fi + +if [[ -r $FU ]]; then + echo "NOTICE: full zip exists: $FU" +else + CC_WHAT="Windows creating full.zip" + #say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT"; + echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + ((cd $IN \ + && unzip -d bin $DEPS 2>&1) 2>&1 >$TO/log/${EB}-unzip-deps.log \ + && echo "3..OK unzip deps") \ + || ( echo "FAILED ($?)" ; exit 3 ) + ((cd $IN \ + && zip -vr9 $FU .) 2>&1 >$TO/log/${EB}-zip.log \ + && echo "4..OK rezip full") \ + || ( echo "FAILED ($?)" ; exit 4 ) +fi + +MAKE_SE=1; +if [[ -r $SE ]] ; then + echo "NOTICE: self-installer exists: $SE"; + if [[ -z $FORCE_SE ]] ; then + # noop? + MAKE_SE=0; + else + echo "FORCE: building SE anyway (FOCE_SE=$FORCE_SE)" + fi +fi +if [[ $MAKE_SE ]]; then + #sleep 10; + if [[ -d $SXS ]] ; then + echo "WARNING: Self-install source dir found, reusing: $SXS"; + sleep 90; + else + mv $IN $SXS || + ( echo "ERROR: installer source mv failed ($?): $IN => $SXS"; + exit 5; ) + fi + + # DIST_SV=$IB/emacs-$EV + # CC_WHAT="Windows: moving installed Emacs to $DIST_SV" + # echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + # sleep 2; + # #rm -rf $DIST_SV; + # mv $IN $DIST_SV; + + CC_WHAT="Windows: creating self-installer" + #say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT"; + echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + ((cd $IB \ + && cp $SOURCE_BMP . \ + && cp $SOURCE_NSI . \ + && cp $SOURCE_COPYING emacs-$EV \ + && makensis -v4 \ + -DEMACS_VERSION=$EV \ + -DVERSION_BRANCH=$EV \ + -DOUT_VERSION=$EV \ + emacs.nsi \ + && mv $OE $SE && mv $IB/$EB $IN) \ + | tee $TO/log/${EB}-esi.log \ + && echo "5..OK executable self installer") \ + || (echo "ERROR: creating self installer ($?)"; exit 5) + + # CC_WHAT="Windows: putting installed Emacs back in $IN" + # echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + # sleep 2; + # mv $DIST_SV $IN; + + # archive self-installer sources + if [[ -d $IN ]] ; then + echo "NOTICE: Found self-installer source archive, leaving: $SXS"; + else + mv $SXS $IN || + ( echo "ERROR: source restore mv failed ($?): $IN => $SXS"; + exit 5; ) + fi +fi + + + +# archive sources +if [[ -r $SZ ]] ; then + echo "NOTICE: source zip exists: $SZ"; +else + CC_WHAT="Windows: creating src archive" + #say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT"; + echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + # clean-up the build folder + cd $SRC + git clean -fxd 2>&1 >$TO/log/${EB}-clea.log + # archive sources, omit git cruft + ((zip -9r $SZ . -x .git/ .git/\* 2>&1 >$TO/log/${EB}-src.log \ + ) && echo "6..OK archive sources" + ) || (echo "ERROR: archive sources ($?)"; exit 4); +fi + +cd $UP + +# create SHA256 sums +if [[ $( ls -t *.{exe,zip,txt} 2>/dev/null | head -1 ) \ + != \ + $( ls *.txt 2>/dev/null ) ]] ; +then + CC_WHAT="Windows: writing SHA256 sums" + #say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT"; + echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + ((for f in *.{zip,exe} ; + do + sha256sum.exe $f ; + done) | tee $UP/${EL}-sha256sums.txt + ) | tee $TO/log/${EB}-sums.log +fi + +# sign release files +EXTS=exe,zip,txt +if [[ $( ls $UP/*.{$EXTS} 2>/dev/null | wc -l ) \ + -ne \ + $( ls $UP/*.sig 2>/dev/null | wc -l) ]] ; +then + CC_WHAT="Windows: signing files" + #say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT"; + echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" + (for f in $UP/*.{txt,exe,zip} ; + do + gpg --pinentry-mode=loopback \ + --passphrase-file=$GPG_PPF \ + --batch --yes -b $f + done) | tee $TO/lop/emacs-${SLUG}-sign.log + exit 0 +fi + +# create upload directives +if [[ $( ls $UP/*.{$EXTS} 2>/dev/null | wc -l ) \ + -ne \ + $( ls $UP/*.directive.asc 2>/dev/null | wc -l) ]] ; +then + (for f in *.{zip,exe,txt} ; + do + cat $DIR \ + | perl -p \ + -e "s/__FILE__/$f/msg;" \ + -e "s/__MAJOR_VERSION__/$MV/msg;" \ + -e "s/__VERSION__/${EV:=$MV.0.50}/msg;" \ + > $f.directive ; + done) | tee $TO/log/emacs-${SLUG}-dirs.log + + # sign directives + (((for f in *.directive ; + do + gpg --pinentry-mode=loopback \ + --passphrase-file=$HOME/emacs-build/foo.txt \ + --batch --yes --clearsign $f ; + done) | tee $TO/log/emacs-${SLUG}-sidr.log + ) && echo "7..OK prep upload" + ) || (echo "ERROR: prep upload ($?)"; exit 4); +fi + +rsync -vvrte "/usr/bin/ssh -i $SSH_KEY" "$UP" "${SSH_USER}@corwin.bru.st:~/corwin-emacs/emacs${EU}-$MV" +ssh -i $SSH_KEY ${SSH_USER}@corwin.bru.st 'cd ~/corwin-emacs/emacs-32; ./update-sym-links.sh' + +CC_WHAT="Windows build complete" +CC_WHAT="$CC_WHAT https://corwin.bru.st/emacs-$MV/emacs-$MV-$SHORT_VER" +say "$SHORT_VER" "$CC_BRANCH" "$CC_WHO" "$CC_WHERE" "$CC_WHAT"; +echo "$(date) NOTICE: [${branch}@${SHORT_VER}:${RC}] $CC_WHO: $CC_WHAT" diff --git a/crude-ci.sh b/crude-ci.sh new file mode 100644 index 0000000..fd5ed9a --- /dev/null +++ b/crude-ci.sh @@ -0,0 +1,142 @@ +#!/usr/bin/bash +# crude-ci - build new Emacs binaries for windows after commits6 +# +# Copyright 2026 Corwin Brust +# +# This program is distributed under the terms of the GNU Public +# License version 3 or (at your option) any later version. +# + +test -n "$DEBUG" && set -x + +# most likely things to edit +SLEEP_TIME=${SLEEP_TIME-17} + +# if truthy, try to rebuild tree-sitter grammers also +# NO_TS=${NO_TS-1} + +# number of loops between tree sitter project checks +TS_WAIT=$((17 * 60)) + +LK_WAIT=$((17 * 60)) + +EC=${EC:-"$( echo "$MSYSTEM" | grep UCRT64 )"} +EU=${EU:-"$(test -n "$EC" && printf '%s' "-ucrt")"} + +# the name the current master branch will have when it becomes the release branch +MASTER=emacs$EU-32 + +# webpage with the head-revision of a given branch +BASE_URI=$( printf '%s://%s/%s?project=%s&property=%s&' \ + 'https' \ + 'test.comic.chat' \ + 'last-rev.pl' \ + 'emacs' \ + 'revision' ) + +# counter +runs_since_last_ts=$TS_WAIT + +LOCKFILE=/g/.crude-ci.lock +if test -r $LOCKFILE ; then + echo "ERROR: cannot start while lockfile exists, try:\n\trm $LOCKFILE"; + exit 1; +fi + +# the .private file exports SSH_KEY and SSH_USER, needed for rsync +if [[ -r /g/.private ]] ; then + . /g/.private +else + echo "Private settings missing"; + exit 1; +fi + +sleep 10; + +while true ; +do + if test -r $LOCKFILE ; then + echo -e "\e[1;6m$( date +'%Y-%m-%d %H:%M' ) Build in process. Trying again in $LK_WAIT seconds..\e[0m" + sleep $LK_WAIT + continue + fi + # maybe rebuild tree-sitter grammers + # NOTE: only build TS grammers under MSYS54 for now + if test -z "$NO_TS" && test -z "$EC" ; then + runs_since_last_ts=$((1+ runs_since_last_ts)) + if [[ $runs_since_last_ts > $TS_WAIT ]] ; then + touch $LOCKFILE + echo -e "\e[1;2m$( date +'%Y-%m-%d %H:%M' ) Checking treesitter projects...\e[0m" + runs_since_last_ts=$((0)); + /g/emacs-local-ci/build-tree-sitter.sh + rm $LOCKFILE + fi + fi + for VER in 32 31 ; + do + branch_folder_name=emacs$EU-$VER + cd /g/git/$branch_folder_name + + # special case name of the master branch + branch_name=emacs-$VER #$branch_folder_name + if [[ $MASTER -eq $branch_folder_name ]] ; then + branch_name=master + branch_color=35 + else + branch_color=36 + fi + + # echo "checking Emacs branche $branch_name for changes..." + + local_rev=$( git rev-parse HEAD ) + echo "Fetching... ${BASE_URI}&branch=${branch_name}" + remote_rev=$( wget -qO - $BASE_URI'&'branch=$branch_name ); + if [[ "$local_rev" != "$remote_rev" ]] ; then + local_color=33 + else + local_color=32 + fi + + echo -e "\e[1;2m$( date +'%Y-%m-%d %H:%M' )\e[0m " \ + "\e[1;${branch_color}m$branch_folder_name\e[0m " \ + "\e[1;2mhave:\e[0m\e[1;32m${local_rev:0:6}\e[0m " \ + "\e[1;2mwant:\e[0m\e[1;${local_color}m${remote_rev:0:6}\e[0m" + + if [[ -n $FORCE ]] || [[ "$local_rev" != "$remote_rev" ]] ; then + + touch $LOCKFILE + # wait, in case of slow trickle of related commits + if [[ -z $FORCE ]] ; then + echo "pausing for $SLEEP_TIME seconds.."; + sleep $SLEEP_TIME; + fi + + echo "GIT: pull --rebase .." + git pull --rebase; + new_rev=$( git rev-parse HEAD ); + new_rev_short=${new_rev:0:6}; + (set -x; + MV=$VER /g/emacs-local-ci/build-dist.sh; + )2>&1 | tee -a /g/log/${branch_folder_name}-$new_rev_short-`date +%Y-%m-%d-%H%M`.log + + problems=`(/g/emacs-local-ci/check-feature.pl ${branch_folder_name}-$new_rev_short; /g/emacs-local-ci/check-log.sh ${branch_folder_name}) | grep NOT | cut -d ' ' -f 1,4 | perl -e 'END{ print join q(, ), @a }' -ne 's/\s+/ /msg; push @a, $_ if length'`; + + if [[ -n $problems ]] ; then + health="dubious: $problems"; + else + health="OK" + fi + + if [[ -n $CC_SSH_KEY ]] && [[ -n $CC_SSH_USER ]] && [[ -n $CC_SSH_HOST ]] ; then + #last_up="$( basename $( ls -1td /h/upload/$branch_folder_name* | head -1 ) )" + BUILD_URI="https://corwin.bru.st/$branch_folder_name/$branch_folder_name-$new_rev_short" + ssh -i "$CC_SSH_KEY" "$CC_SSH_USER@$CC_SSH_HOST" \ + "echo '#emacs-dev [$branch_name@$new_rev_short] `uname -o` build $health $BUILD_URI (`hostname`)' >>/petroglyph/udp2irc/gliphy.fifo" + fi + rm $LOCKFILE + fi + done + FORCE= + powershell /g/emacs-local-ci/fix-shortcut.ps1 + sleep ${SLEEP_TIME} +done