Make packaging more generic

Move specifics into a subdirectory and detect them
Hooks detect if there is a command to run
Add release notes to README.md
This commit is contained in:
Philip Whineray 2015-11-28 09:33:31 +00:00
parent 7f76c312aa
commit da346f605f
11 changed files with 519 additions and 161 deletions

@ -1,2 +1,5 @@
#!/bin/sh
exec git diff HEAD^ | ./packaging/update-tags -
if [ -x ./packaging/update-tags ]
then
exec git diff HEAD^ | ./packaging/update-tags -
fi

@ -1,2 +1,5 @@
#!/bin/sh
exec git diff --cached | ./packaging/check-files -
if [ -x ./packaging/check-files ]
then
exec git diff --cached | ./packaging/check-files -
fi

@ -1,2 +0,0 @@
The programs in this folder are used when packaging from within git
and are not included in source or binary packages.

49
packaging/README.md Normal file

@ -0,0 +1,49 @@
Packaging Tools
===============
The programs in this folder are used when packaging from within git
and are not included in source or binary packages.
For the most part they are used from the git commit hooks (copy
`../hooks/*` to `../.git/hooks` to automate checking and the release
process.
The check-files script pulls in `*.functions` and `*/*.functions` to
do the actual work.
`packaging.functions` contains generic checks on e.g `ChangeLog`
and `configure.ac` and automates release version, checking, tagging
and post-release update.
Programs and packages with specific needs should create extra
`whatever.functions` and supporting scripts in a subdirectory.
Making a release
----------------
`
Just update ChangeLog and configure.ac to specify a suitable version
suffix:
empty - final release
pre.# - pre-release candidate
rc.# - pre-release candidate
The hooks will take over and if everything is OK will tag the release
(you will be asked to sign the tag) and then update the files ready
for further development.
The release is not pushed out automatically, so if you want to undo
it, run:
~~~~
git reset --hard HEAD^^
git tag -d vx.y.z
~~~~
Otherwise you can just push the results; the script outputs the required
instructions upon success.
Once pushed the infrastructure will build a set of tar-files on the server.
For information on how to verify, sign and make these available, see:
https://github.com/firehol/infrastructure/raw/master/doc/release.txt

@ -87,59 +87,40 @@ fi
dirname="${0%/*}"
if [ "$dirname" = "$0" ]; then dirname="."; fi
source $dirname/packaging.functions
for i in $dirname/*.functions $dirname/*/*.functions
do
if [ -f "$i" ]
then
source $i
echo $i | sed -e 's:.*/::' -e 's/\.functions$//' -e 's/\./_/g' >> $MYTMP/fns
fi
done
status=0
while read fn
do
"${fn}_check_init" $filename || status=1
done < $MYTMP/fns
while read filename
do
#echo Checking $filename
case $filename in
sbin/Makefile.in)
:
;;
configure.ac|sbin/commands.sed.in)
check_versions || status=1
version_check=Y
if [ $status -eq 0 ]
then
check_commands sbin/firehol.in || status=1
check_commands sbin/fireqos.in || status=1
check_commands sbin/link-balancer.in || status=1
check_commands sbin/update-ipsets.in || status=1
check_commands sbin/vnetbuild.in || status=1
fi
;;
sbin/*.in)
check_commands $filename || status=1
;;
ChangeLog)
check_versions || status=1
version_check=Y
if [ $status -eq 0 ]
then
check_changelog || status=1
fi
;;
*)
#echo "No checks found for $filename"
:
;;
esac
while read fn
do
if [ $status -eq 0 ]
then
"${fn}_check_file" $filename || status=1
fi
done < $MYTMP/fns
done < $MYTMP/files.lst
if [ $status -eq 0 -a "$version_check" ]
if [ $status -eq 0 ]
then
version=`get_configure_ac_version`
case "$(match_version $version)" in
prerelease|candidate|release)
try_build || status=1
if [ $status -eq 0 ]
then
cp $MYTMP/build/*.tar.* .
echo "Check $(cd $MYTMP/build && ls *.tar.gz) before pushing tags"
fi
;;
esac
while read fn
do
"${fn}_check_fin" $filename || status=1
done < $MYTMP/fns
fi
exit $status

@ -1,11 +1,16 @@
#!/usr/bin/perl -w
#
# Find FireHOL program lines that are using program names direct rather
# than via detected environments
#
use strict;
use Data::Dumper;
use File::Basename;
if (@ARGV == 0) {
print "Usage: ./packaging/detect-cmd.pl configure.ac sbin/file.in ...\n";
print "Usage: ./packaging/firehol/detect-cmd.pl configure.ac sbin/file.in ...\n";
print "\n";
print "Finds usages of commands which should be converted to \$COMMAND_CMD format\n";
exit 0;

@ -0,0 +1,376 @@
#!/bin/bash
firehol_check_init() {
:
}
firehol_check_file() {
local filename="$1" status=0
case $filename in
sbin/Makefile.in)
:
;;
configure.ac|sbin/commands.sed.in)
check_commands sbin/firehol.in || status=1
check_commands sbin/fireqos.in || status=1
check_commands sbin/link-balancer.in || status=1
check_commands sbin/update-ipsets.in || status=1
check_commands sbin/vnetbuild.in || status=1
;;
sbin/*.in)
check_commands $filename || status=1
;;
*)
#echo "No checks found for $filename"
:
;;
esac
return $status
}
firehol_check_fin() {
:
}
get_staged_file() {
local dir="${1%/*}"
if [ "$dir" = "$1" ]; then dir="."; fi
mkdir -p $MYTMP/files/$dir
if [ "$from_cache" ]
then
test -f $MYTMP/files/$1 || git show :$1 > $MYTMP/files/$1
else
test -f $MYTMP/files/$1 || cp $1 $MYTMP/files/$1
fi
}
try_build() {
if [ -f $MYTMP/success ]; then return 0; fi
mkdir -p $MYTMP/build
git archive HEAD | tar -xf - -C "$MYTMP/build"
git diff --staged | patch -p1 -d "$MYTMP/build"
(cd $MYTMP/build; ./packaging/git-build || touch $MYTMP/fail)
if [ -f $MYTMP/fail ]; then return 1; fi
(cd $MYTMP/build; ./packaging/tar-compare . *.tar.gz || touch $MYTMP/fail)
if [ -f $MYTMP/fail ]; then return 1; fi
touch $MYTMP/success
return 0
}
get_changelog_version() {
get_staged_file ChangeLog
local v=`sed -ne '1s/.*(\(.*\)).*/\1/p' $MYTMP/files/ChangeLog`
if [ ! "$v" ]; then v="No version in ChangeLog!"; fi
echo "$v"
}
get_configure_ac_version() {
get_staged_file configure.ac
local v=`sed -n \
-e '/define(\[VERSION_\(MINOR\|FIX\)/s/.*\[\([^[]*\)\].*/.\1/p' \
-e '/define(\[VERSION_\(MAJOR\|SUFFIX\)/s/.*\[\([^[]*\)\].*/\1/p' \
$MYTMP/files/configure.ac | tr -d '\n'`
if [ ! "$v" ]; then v="No version in configure.ac!"; fi
echo "$v"
}
splitver() {
local maj min fix sfx IFS=.-_
maj=$1
min=$2
fix=$3
sfx=$4
set -- $5
eval $maj=\$1 $min=\$2 $fix=\$3
}
match_version() {
case "$1" in
[0-9]*.[0-9]*.[0-9]*_*)
# x.y.z_ZZZZ = development branch (can be before or after pre/rc)
echo "development"
;;
[0-9]*.[0-9]*.[0-9]-pre[0-9]*)
echo "prerelease"
;;
[0-9]*.[0-9]*.[0-9]-rc.[0-9]*)
echo "candidate"
;;
[0-9]*.[0-9]*.[0-9]*)
echo "release"
;;
*)
# Unknown
:
;;
esac
}
check_versions() {
local status=0 exact=0 prerelease=0
if [ -f $MYTMP/version-checked ]
then
read status < $MYTMP/version-checked
return $status
fi
local confver=`get_configure_ac_version`
case "$(match_version $confver)" in
development)
:
;;
prerelease|candidate|release)
exact=1
;;
*)
echo "Unrecognised version in configure.ac ($confver)"
status=1
;;
esac
local clogver=`get_changelog_version`
case "$(match_version $clogver)" in
development)
echo "Do not include development branch version in ChangeLog ($clogver)"
status=1
;;
prerelease|candidate)
prerelease=1
;;
release)
:
;;
*)
echo "Unrecognised version format in ChangeLog ($clogver)"
status=1
;;
esac
if [ $status -eq 0 -a $exact -eq 0 ]
then
local clogmaj clogmin clogfix clogsfx
local confmaj confmin conffix confsfx
splitver clogmaj clogmin clogfix clogsfx "$clogver"
splitver confmaj confmin conffix confsfx "$confver"
if [ $confmaj -gt $clogmaj \
-o $confmin -gt $clogmin \
-o $conffix -gt $clogfix ]
then
:
elif [ $confmaj -eq $clogmaj \
-a $confmin -eq $clogmin \
-a $conffix -eq $clogfix \
-a $prerelease -eq 1 ]
then
:
else
echo "Version in configure.ac ($confver) lesser than ChangeLog ($clogver)"
status=1
fi
fi
if [ $exact -eq 1 ]
then
echo "Running additional release checks"
if [ "$confver" != "$clogver" ]
then
echo "Version in configure.ac ($confver) differs from ChangeLog ($clogver)"
status=1
elif [ "$(git tag -l v$confver)" ]
then
echo "Tag v$confver already exists"
status=1
fi
if [ "$(git config user.signingkey)" = "" ]
then
echo "You need to set up a PGP signing key e.g.:"
echo " gpg --list-keys"
echo "and"
echo " git config user.signingkey SHORTID"
echo "or"
echo " git config --global user.signingkey SHORTID"
status=1
fi
git status -s | grep "^?" > $MYTMP/needclean
if [ -s $MYTMP/needclean ]
then
echo "The following files must be dealt with before commit:"
cat $MYTMP/needclean
echo "e.g. add them to .gitignore or remove with 'git clean -fdx'"
status=1
fi
fi
echo $status > $MYTMP/version-checked
return $status
}
check_commands() {
local status=0
touch $MYTMP/commands-checked
if grep -q -F -z "$1" $MYTMP/commands-checked
then
# Only check a file once - an edit to some files checks multiple
return 0
else
echo "$1" >> $MYTMP/commands-checked
fi
get_staged_file $1
get_staged_file configure.ac
get_staged_file sbin/commands.sed.in
get_staged_file sbin/functions.common.sh
# Find commands that have been enclosed in quotes and remove anything after
# if nothing matched the substitution, proceed to the next line
# if the command is used in a -z check, proceed to the next line
# if the command is in posix size check [ "$var" ], proceed to the next line
# if the command is being assigned to a variable, proceed to the next line
# if the command is being used in an eval, proceed to the next line
# Anything left, print the line number
sed -n -e 's/.*\([(`|&][^)`]*"\$[^("}]*_CMD}*\).*/\1/' \
-e 's/^[ ]*\("\$[^("}]*_CMD}*\)/\1/' \
-e T \
-e '/-z *"\$/b' \
-e '/\[ *"\$/b' \
-e '/= *"\$/b' \
-e '/eval *"\$/b' \
-e '{=}' \
$MYTMP/files/$1 > $MYTMP/errors
if [ -s $MYTMP/errors ]
then
status=1
echo "Detected use(s) of \"\${SOMETHING_CMD}\" in ${1}. Check lines:"
cat $MYTMP/errors
fi
# Ignore lines from the command table and require_cmd entries
# ensure at least 2 characters before and after each word, reset cond jump
# extract the command and immediate surroundings, got to next line if no match
# remove brackets
# proceed to next line if command starts with a dollar
# proceed to next line if command ends with equals (assignment)
# print line number
sed -n -e '/^ *[YN]|/b' -e '/require_cmd/b' \
-e 's/^/ /' -e 's/$/ /' -e T \
-e 's/.*\(..\<[A-Z0-9_]*_CMD..\).*/\1/' -e T \
-e 's/[{}]//' \
-e '/$[A-Z]/b' \
-e '/[A-Z0-9]=/b' \
-e '=' \
$MYTMP/files/$1 > $MYTMP/errors
if [ -s $MYTMP/errors ]
then
status=1
echo "Detected use(s) of 'SOMETHING_CMD' (missing \$) in ${1}. Check lines:"
cat $MYTMP/errors
fi
sed -n -e "s/^ *[YN]|//p" $MYTMP/files/$1 > $MYTMP/commands-defined
for cmd in $(tr " " "\n" < $MYTMP/files/$1 |
sed -n -e 's/.*\(\<[A-Z0-9_]*\)_CMD.*/\1/p' | sort | uniq)
do
if ! grep -q "^${cmd}_CMD|" $MYTMP/commands-defined
then
status=1
echo "Missing definition of $cmd in $1 detection table."
fi
# Hostname is a special case - configure will expand it to running host,
# overwriting the value we wanted to use.
if [ "$cmd" = "HOSTNAME" ]; then cmd="HOSTNAMECMD"; fi
if ! grep -q "_${cmd}(\|\[$cmd\]" $MYTMP/files/configure.ac
then
status=1
echo "Missing detection of $cmd for $1 in configure.ac"
fi
done
while IFS="|" read cmd subst defaults
do
if ! grep -q "\${*$cmd" $MYTMP/files/$1 $MYTMP/files/sbin/*.sh
then
status=1
echo "$cmd detected but never used in $1 or function libraries"
fi
if ! grep -q "#$subst#" $MYTMP/files/sbin/commands.sed.in
then
status=1
echo "$cmd detected but $subst never substituted by sbin/commands.sed.in"
fi
done < $MYTMP/commands-defined
(
a=`pwd`
cd $MYTMP/files
$a/packaging/firehol/detect-cmd.pl configure.ac $1 || touch $MYTMP/detect
)
if [ -f $MYTMP/detect ]; then status=1; fi
return $status
}
check_changelog() {
local status=0
get_staged_file ChangeLog
local version=`sed -ne '1s/.*(\(.*\)).*/\1/p' $filename`
if [ "`echo $version | grep '[[:space:]]'`" != "" ]
then
echo "ChangeLog version contains whitespace! Fix it!"
status=1
fi
set +e
which dpkg-gencontrol > /dev/null 2>&1
status=$?
set -e
if [ $status -ne 0 ]
then
echo "dpkg-gencontrol not found; skipping ChangeLog verification!"
return $status
fi
(cd $MYTMP/files
mkdir -p debian/tmp
cat > debian/control <<!
Source: firehol
Section: net
Priority: optional
Homepage: http://firehol.org/
Maintainer: firehol-devs@lists.firehol.org
Package: firehol
Architecture: all
Depends: iptables
Description: firewall for humans
!
set +e
dpkg-gencontrol -O -pfirehol -lChangeLog > /dev/null 2> debian/tmp/l
set -e
status=$?
if [ $status -eq 0 -a `grep -v "FcntlLock not available" debian/tmp/l | wc -c` -gt 0 ]
then
status=1
fi
if [ $status -ne 0 ]
then
cat debian/tmp/l
touch $MYTMP/changelog.fail
fi
)
if [ -f $MYTMP/changelog.fail ]; then status=1; fi
return $status
}

@ -1,14 +1,14 @@
#!/usr/bin/perl -w
# Create the configure.ac lines necessary to detect external commands
# used by the programs.
# used by the FireHOL programs.
use strict;
use Data::Dumper;
use File::Basename;
if (@ARGV == 0) {
print "Usage: ./packaging/gen-config-detect.pl sbin/file.in ...\n";
print "Usage: ./packaging/firehol/gen-config-detect.pl sbin/file.in ...\n";
print "\n";
print "Generates the configure.ac content for detecting commands\n";
exit 0;

@ -3,9 +3,9 @@
# When run from the top-level repository, performs a complete clean
# and maintainer-mode rebuild of the FireHOL package.
if [ ! -f .gitignore -o ! -f sbin/firehol.in ]
if [ ! -f .gitignore -o ! -f configure.ac -o ! -x autogen.sh ]
then
echo "Run as ./packaging/git-build from a firehol git repository"
echo "Run as ./packaging/git-build from an autotools git repository"
exit 1
fi

@ -1,5 +1,50 @@
#!/bin/bash
packaging_check_init() {
version_check=
}
packaging_check_file() {
local filename="$1" status=0
case $filename in
configure.ac)
check_versions || status=1
version_check=Y
;;
ChangeLog)
check_versions || status=1
version_check=Y
if [ $status -eq 0 ]
then
check_changelog || status=1
fi
;;
*)
#echo "No checks found for $filename"
:
;;
esac
return $status
}
packaging_check_fin() {
if [ "$version_check" ]
then
version=`get_configure_ac_version`
case "$(match_version $version)" in
prerelease|candidate|release)
try_build || status=1
if [ $status -eq 0 ]
then
cp $MYTMP/build/*.tar.* .
echo "Check $(cd $MYTMP/build && ls *.tar.gz) before pushing tags"
fi
;;
esac
fi
}
get_staged_file() {
local dir="${1%/*}"
if [ "$dir" = "$1" ]; then dir="."; fi
@ -181,112 +226,6 @@ check_versions() {
return $status
}
check_commands() {
local status=0
touch $MYTMP/commands-checked
if grep -q -F -z "$1" $MYTMP/commands-checked
then
# Only check a file once - an edit to some files checks multiple
return 0
else
echo "$1" >> $MYTMP/commands-checked
fi
get_staged_file $1
get_staged_file configure.ac
get_staged_file sbin/commands.sed.in
get_staged_file sbin/functions.common.sh
# Find commands that have been enclosed in quotes and remove anything after
# if nothing matched the substitution, proceed to the next line
# if the command is used in a -z check, proceed to the next line
# if the command is in posix size check [ "$var" ], proceed to the next line
# if the command is being assigned to a variable, proceed to the next line
# if the command is being used in an eval, proceed to the next line
# Anything left, print the line number
sed -n -e 's/.*\([(`|&][^)`]*"\$[^("}]*_CMD}*\).*/\1/' \
-e 's/^[ ]*\("\$[^("}]*_CMD}*\)/\1/' \
-e T \
-e '/-z *"\$/b' \
-e '/\[ *"\$/b' \
-e '/= *"\$/b' \
-e '/eval *"\$/b' \
-e '{=}' \
$MYTMP/files/$1 > $MYTMP/errors
if [ -s $MYTMP/errors ]
then
status=1
echo "Detected use(s) of \"\${SOMETHING_CMD}\" in ${1}. Check lines:"
cat $MYTMP/errors
fi
# Ignore lines from the command table and require_cmd entries
# ensure at least 2 characters before and after each word, reset cond jump
# extract the command and immediate surroundings, got to next line if no match
# remove brackets
# proceed to next line if command starts with a dollar
# proceed to next line if command ends with equals (assignment)
# print line number
sed -n -e '/^ *[YN]|/b' -e '/require_cmd/b' \
-e 's/^/ /' -e 's/$/ /' -e T \
-e 's/.*\(..\<[A-Z0-9_]*_CMD..\).*/\1/' -e T \
-e 's/[{}]//' \
-e '/$[A-Z]/b' \
-e '/[A-Z0-9]=/b' \
-e '=' \
$MYTMP/files/$1 > $MYTMP/errors
if [ -s $MYTMP/errors ]
then
status=1
echo "Detected use(s) of 'SOMETHING_CMD' (missing \$) in ${1}. Check lines:"
cat $MYTMP/errors
fi
sed -n -e "s/^ *[YN]|//p" $MYTMP/files/$1 > $MYTMP/commands-defined
for cmd in $(tr " " "\n" < $MYTMP/files/$1 |
sed -n -e 's/.*\(\<[A-Z0-9_]*\)_CMD.*/\1/p' | sort | uniq)
do
if ! grep -q "^${cmd}_CMD|" $MYTMP/commands-defined
then
status=1
echo "Missing definition of $cmd in $1 detection table."
fi
# Hostname is a special case - configure will expand it to running host,
# overwriting the value we wanted to use.
if [ "$cmd" = "HOSTNAME" ]; then cmd="HOSTNAMECMD"; fi
if ! grep -q "_${cmd}(\|\[$cmd\]" $MYTMP/files/configure.ac
then
status=1
echo "Missing detection of $cmd for $1 in configure.ac"
fi
done
while IFS="|" read cmd subst defaults
do
if ! grep -q "\${*$cmd" $MYTMP/files/$1 $MYTMP/files/sbin/*.sh
then
status=1
echo "$cmd detected but never used in $1 or function libraries"
fi
if ! grep -q "#$subst#" $MYTMP/files/sbin/commands.sed.in
then
status=1
echo "$cmd detected but $subst never substituted by sbin/commands.sed.in"
fi
done < $MYTMP/commands-defined
(
a=`pwd`
cd $MYTMP/files
$a/packaging/detect-cmd.pl configure.ac $1 || touch $MYTMP/detect
)
if [ -f $MYTMP/detect ]; then status=1; fi
return $status
}
check_changelog() {
local status=0

@ -42,14 +42,18 @@ diff -r "$1" $MYTMP/unpack/* | grep "^Only" | sed \
-e '/: \.gitignore$/d' \
-e '/: config\.log$/d' \
-e '/: config\.status$/d' \
-e '/: config\.h.*$/d' \
-e '/: Makefile$/d' \
-e '/: hooks$/d' \
-e '/: packaging$/d' \
-e '/: stamp-h1$/d' \
-e '/: README\.md$/d' \
-e '/: tmp-anchor-links$/d' \
-e '/: tmp-manproc$/d' \
-e '/: firehol.*\.tar\.\(gz\|bz2\|xz\)$/d' \
-e '/: .*\.tar\.\(gz\|bz2\|xz\)$/d' \
-e '/: unittest$/d' \
-e '/: iprange$/d' \
-e '/: .*\.o$/d' \
-e '/sbin: \(firehol\|fireqos\|link-balancer\)$/d' \
-e '/sbin: \(update-ipsets\|vnetbuild\|commands.sed\)$/d' > $MYTMP/out