From f83a3809e48a69427877e251895627e32698d74a Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Thu, 1 Jan 1970 00:00:00 +0000 Subject: [PATCH] Rewrite smock in Perl, can work with multiple SRPMs and auto-analyze the dependencies to produce a correct build order. Rewrite top-level scripts to use the new smock. --- .hgignore | 2 +- README | 7 -- build-everything-in-mock.sh | 64 ++++------- show-build-order.pl | 176 ------------------------------ smock/README | 9 +- smock/smock.pl | 255 ++++++++++++++++++++++++++++++++++++++++++++ smock/smock.sh | 69 ------------ 7 files changed, 285 insertions(+), 297 deletions(-) delete mode 100755 show-build-order.pl create mode 100755 smock/smock.pl delete mode 100755 smock/smock.sh diff --git a/.hgignore b/.hgignore index 9165b23..f802fc6 100644 --- a/.hgignore +++ b/.hgignore @@ -5,7 +5,7 @@ syntax: glob *.cmi *.cmx *.o -buildall.log +*.src.rpm SDL/SDL-1.2.13.tar.gz atk/atk-1.23.5.tar.bz2 diff --git a/README b/README index af8fd67..4c9292c 100644 --- a/README +++ b/README @@ -69,10 +69,3 @@ Package notes Note that once built and installed, these last two replace the files built from binaries in mingw32-{runtime,w32api}-bootstrap. - -Then for the rest, use ./show-build-order.pl which is a script which -works out the correct order to build packages and will display the -list of commands that you have to invoke to do this. - -(Note that show-build-order.pl doesn't actually build anything - it -just prints suggested commands). diff --git a/build-everything-in-mock.sh b/build-everything-in-mock.sh index e8bdec3..b36a81e 100755 --- a/build-everything-in-mock.sh +++ b/build-everything-in-mock.sh @@ -1,45 +1,27 @@ #!/bin/bash - -DIST=fedora-9 -SKIP_BUILT_RPMS=1 - -LOCALREPO=$HOME/public_html/smock/yum -ARCHES="i386 x86_64" - -export DIST SKIP_BUILT_SRPMS LOCALREPO ARCHES - -specs=`perl show-build-order.pl | - grep -v '^#' | - grep -Eo '[^[:space:]]+/mingw32-[^[:space:]]+\.spec'` - -rm -f buildall.log -echo -e "Specfiles in build order:\n$specs\n\n" >> buildall.log - -pwd=`pwd` - -for spec in $specs -do - set -e - dir=`dirname $spec` - srcrpm=`rpmbuild --define "_sourcedir $pwd/$dir" -bs $spec` - if [ $? != 0 ]; then exit 1; fi - srcrpm=`echo $srcrpm | awk '{print $2}'` - - # Test if all the output RPMs exist already. - skip= - if [ $SKIP_BUILT_RPMS ]; then - skip=1 - baserpm=`basename $srcrpm | sed 's/\.fc[[:digit:]]*\.src\.rpm//g'` - for arch in $ARCHES; do - if [ ! -f $LOCALREPO/$DIST/$arch/RPMS/$baserpm.* ]; then - skip= - fi - done +# These are the packages we don't want to build yet: +nobuild="example +cyrus-sasl +gdb +pidgin +python +nspr +nss +ocaml-lablgl +wix" + +rm -f */*.src.rpm + +for dir in *; do + if ! echo "$nobuild" | grep -sq "^$dir\$"; then + if [ -d $dir -a -f $dir/*.spec ]; then + ( + cd $dir + rpmbuild -bs --define "_sourcedir $(pwd)" --define "_srcrpmdir $(pwd)" *.spec + ) + fi fi +done - if [ $skip ]; then - echo "skipping $srcrpm" - else - smock/smock.sh $DIST $srcrpm - fi -done 2>&1 | tee -a buildall.log +smock/smock.pl --arch=i386 --arch=x86_64 --distro=fedora-10 */*.src.rpm diff --git a/show-build-order.pl b/show-build-order.pl deleted file mode 100755 index 2973fc0..0000000 --- a/show-build-order.pl +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/perl -w -# -# Show the order to build Fedora MinGW spec files. -# By Richard Jones - -use strict; - -my $debug = 0; -chomp (my $pwd = `pwd`); - -sub main { - my %br; - my $specfile; - my $packagename; - - my @specfiles = <*/*.spec>; - - # Get BRs for each specfile. - foreach $specfile (@specfiles) { - $packagename = $specfile; - $packagename =~ s{^.*/}{}; - $packagename =~ s{\.spec$}{}; - - $br{$packagename} = []; - - open SPEC,$specfile or die "$specfile: $!"; - while () { - if (m/^BuildRequires:(.*)/) { - my $brs = $1; - my @brs = eval 'split /,/, $brs'; - @brs = map { trim ($_) } @brs; - @brs = map { remove_trailers ($_) } @brs; - unshift @{$br{$packagename}}, @brs; - } - } - - if ($debug) { - print "BRs for $packagename = ["; - print (join "],[", @{$br{$packagename}}); - print "]\n"; - } - } - - foreach $packagename (keys %br) { - my @brs = @{$br{$packagename}}; - @brs = uniq (sort @brs); - $br{$packagename} = \@brs; - - if ($debug) { - print "uniq BRs for $packagename = ["; - print (join "],[", @{$br{$packagename}}); - print "]\n"; - } - } - - # Some packages we want to ignore for now. - delete $br{"mingw32-cyrus-sasl"}; - delete $br{"mingw32-wix"}; - delete $br{"mingw32-example"}; - delete $br{"mingw32-gdb"}; - delete $br{"mingw32-python"}; - delete $br{"mingw32-pidgin"}; - delete $br{"mingw32-nspr"}; - delete $br{"mingw32-nss"}; - delete $br{"mingw32-ocaml-lablgl"}; - - # There is a dependency loop (gcc -> runtime/w32api -> gcc) - # which has to be manually resolved below. Break that loop. - my @gcc_brs = @{$br{"mingw32-gcc"}}; - @gcc_brs = grep { $_ ne "mingw32-runtime" && $_ ne "mingw32-w32api" } @gcc_brs; - $br{"mingw32-gcc"} = \@gcc_brs; - - # Use tsort to generate a topological ordering. - open TSORT,">/tmp/tsort.tmp" or die "/tmp/tsort.tmp: $!"; - foreach $packagename (keys %br) { - my $br; - foreach $br (@{$br{$packagename}}) { - print "writing $br $packagename\n" if $debug; - print TSORT $br, " ", $packagename, "\n"; - } - } - close TSORT; - - system ("tsort < /tmp/tsort.tmp > /tmp/tsort2.tmp") == 0 - or die "system: tsort: $?"; - - # Read in list of packages. - open PACKAGES,"/tmp/tsort2.tmp" or die "/tmp/tsort2.tmp: $!"; - unless ($debug) { - unlink "/tmp/tsort.tmp"; - unlink "/tmp/tsort2.tmp"; - } - - my %installed; - - while () { - chomp; - if (/^mingw32-(.*)/ && exists $br{$_}) { - $packagename = $_; - my $dirname = $1; - - print "considering $packagename\n" if $debug; - - my @brs = @{$br{$packagename}}; - - # Are all BR RPMs installed? - my $br; - foreach $br (@brs) { - if (! rpm_installed ($br) && !exists $installed{$br}) { - print "# as root: rpm -Uvh $br*.rpm\n"; - $installed{$br} = 1; - } - } - - # Special case for mingw32-gcc deps. - if ($packagename eq "mingw32-gcc" && - (!rpm_installed ("mingw32-runtime") || - !rpm_installed ("mingw32-w32api"))) { - print "rpmbuild -ba --define \"_sourcedir $pwd/runtime-bootstrap\" runtime-bootstrap/mingw32-runtime-bootstrap.spec\n"; - print "# as root: rpm -Uvh mingw32-runtime-bootstrap*.rpm\n"; - $installed{"mingw32-runtime-bootstrap"} = 1; - - print "rpmbuild -ba --define \"_sourcedir $pwd/w32api-bootstrap\" w32api-bootstrap/mingw32-w32api-bootstrap.spec\n"; - print "# as root: rpm -Uvh mingw32-w32api-bootstrap*.rpm\n"; - $installed{"mingw32-w32api-bootstrap"} = 1; - } - - # Spec file. - my $specfile = "$dirname/$packagename.spec"; - die "$specfile: file missing" unless -f $specfile; - - my $rpmbuild = - "rpmbuild -ba --define \"_sourcedir $pwd/$dirname\""; - print "$rpmbuild $specfile\n"; - } - } -} - -sub rpm_installed { - local $_ = shift; - return (system ("rpm -q $_ > /dev/null") == 0); -} - -sub trim { - local $_ = shift; - s/^\s+//; - s/\s+$//; - return $_; -} - -sub uniq { - my %hash; - local $_; - - $hash{$_} = 1 foreach (@_); - return sort keys %hash; -} - -# foo >= 3.1 --> foo -# foo-devel --> foo -# and a few other exceptions -sub remove_trailers { - local $_ = shift; - s/\s*[<>=].*$//; - - # -devel & -doc come from the base package. - s/-devel$//; - s/-doc$//; - - # mingw32-gcc-c++ etc. - s/^mingw32-gcc-.*/mingw32-gcc/; - - return $_; -} - -&main() diff --git a/smock/README b/smock/README index 54e923d..e858f15 100644 --- a/smock/README +++ b/smock/README @@ -1,5 +1,6 @@ SMOCK - Simpler Mock ==================== +by Dan Berrange and Richard W.M. Jones. Smock is a thin wrapper around mock to let you build up a whole set of dependant RPMs against an external distro. @@ -20,10 +21,12 @@ set of dependant RPMs against an external distro. Now you can run - ./smock.sh fedora-9 /path/to/srpm + ./smock.pl --arch=i386 --arch=x86_64 --distro=fedora-9 list of srpms -And it'll build the RPM against the fedora-9-XXX distro for each 'XXX' -arch you listed. +And it'll build the all the SRPMs listed on the command line, using +previously built SRPMs as dependencies for later ones. You don't need +to list them in the proper order - the build order is worked out using +the dependencies. The resulting src RPMs, binary RPMs and build logs wil be put into $HOME/public_html/smock, and a Yum repo created. Further RPMs you diff --git a/smock/smock.pl b/smock/smock.pl new file mode 100755 index 0000000..b33d07c --- /dev/null +++ b/smock/smock.pl @@ -0,0 +1,255 @@ +#!/usr/bin/perl -w +# +# SMOCK - Simpler Mock +# by Dan Berrange and Richard W.M. Jones. + +use strict; + +use Getopt::Long; +use Pod::Usage; +use File::Temp qw(tempfile); + +my @arches = (); +my @distros = (); +my $localrepo = $ENV{HOME} . "/public_html/smock/yum"; +my $help = 0; +my $man = 0; + +GetOptions ( + "arch=s" => \@arches, + "distro=s" => \@distros, + "localrepo=s" => \$localrepo, + "help|?" => \$help, + "man" => \$man + ) or pod2usage (2); +pod2usage (1) if $help; +pod2usage (-exitstatus => 0, -verbose => 2) if $man; + +=pod + +=head1 NAME + + smock - Simpler mock + +=head1 SYNOPSIS + + smock.pl --arch=i386 --arch=x86_64 --distro=fedora-10 list of SRPMs ... + +=head1 DESCRIPTION + +This is a wrapper around I which lets you build a whole group of +mutually dependent SRPMs in one go. + +The smock command will work out the correct order in which to build +the SRPMs, and makes the result of previous RPM builds available as +dependencies for later builds. + +Smock also works incrementally. It won't rebuild RPMs which were +built already in a previous run, which means if a package fails to +build, you can just fix it and rerun the same smock command. (In the +unlikely case that you want to force smock to rebuild RPMs then you +must bump the release number or delete the binary RPM from the +localrepo directory). + +B Please read the README file first. You need to set up mock +and a web server before you can use this command. + +=head1 OPTIONS + +=over 4 + +=item B<--arch> + +Specify the architecture(s) to build, eg. i386, x86_64. You can +list this option several times to build several architectures. + +=item B<--distro> + +Specify the distribution(s) to build, eg. fedora-9, fedora-10. +You can list this option several times to build several distributions. + +=item B<--localrepo> + +Local repository. Defaults to C<$HOME/public_html/smock/yum> + +=back + +=cut + +my @srpms = @ARGV; + +if (0 == @arches) { + die "smock: specify one or more architectures using --arch=\n" +} + +if (0 == @distros) { + die "smock: specify one or more distros using --distro=\n" +} + +if (0 == @srpms) { + die "smock: specify one or more SRPMs to build on the command line\n" +} + +# Resolve the names, dependency list, etc. of the SRPMs that were +# specified. + +sub get_one_line +{ + open PIPE, "$_[0] |" or die "$_[0]: $!"; + my $line = ; + chomp $line; + close PIPE; + return $line; +} + +sub get_lines +{ + local $_; + open PIPE, "$_[0] |" or die "$_[0]: $!"; + my @lines; + foreach () { + chomp; + push @lines, $_; + } + close PIPE; + return @lines; +} + +my %srpms = (); +foreach my $srpm (@srpms) { + my $name = get_one_line "rpm -q --qf '%{name}' -p '$srpm'"; + my $version = get_one_line "rpm -q --qf '%{version}' -p '$srpm'"; + my $release = get_one_line "rpm -q --qf '%{release}' -p '$srpm'"; + + my @buildrequires = get_lines "rpm -q --requires -p '$srpm' | + grep -Eo '^[^[:space:]]+'"; + + #print "Filename: $srpm\n"; + #print " name = $name\n"; + #print " version = $version\n"; + #print " release = $release\n"; + #print " buildrequires = ", join (",", @buildrequires), "\n"; + + $srpms{$name} = { + name => $name, + version => $version, + release => $release, + buildrequires => \@buildrequires, + filename => $srpm + } +} + +# We don't care about buildrequires unless they refer to other +# packages that we are building. So filter them on this condition. + +sub is_member_of +{ + my $item = shift; + + foreach (@_) { + return 1 if $item eq $_; + } + 0; +} + +my @names = keys %srpms; +foreach my $name (@names) { + my @buildrequires = @{$srpms{$name}->{buildrequires}}; + @buildrequires = grep { is_member_of ($_, @names) } @buildrequires; + $srpms{$name}{buildrequires} = \@buildrequires; +} + +# Now sort the SRPMs into the correct order for building + +my ($fh, $filename) = tempfile (); + +foreach my $name (@names) { + my @buildrequires = @{$srpms{$name}->{buildrequires}}; + foreach (@buildrequires) { + print $fh "$_ $name\n" + } +} +close $fh; + +my @buildorder = get_lines "tsort $filename"; + +#foreach (@buildorder) { +# print "$_\n"; +#} + +# Now we can build each SRPM. + +sub my_mkdir +{ + local $_ = $_[0]; + + if (! -d $_) { + mkdir ($_, 0755) or die "mkdir $_: $!" + } +} + +sub createrepo +{ + my $arch = shift; + my $distro = shift; + + my_mkdir "$localrepo/$distro"; + my_mkdir "$localrepo/$distro/src"; + my_mkdir "$localrepo/$distro/src/SRPMS"; + system ("cd $localrepo/$distro/src && rm -rf repodata && createrepo -q .") == 0 + or die "createrepo failed: $?\n"; + + my_mkdir "$localrepo/$distro/$arch"; + my_mkdir "$localrepo/$distro/$arch/RPMS"; + my_mkdir "$localrepo/$distro/$arch/logs"; + + system ("cd $localrepo/$distro/$arch && rm -rf repodata && createrepo -q --exclude 'logs/*rpm' .") == 0 + or die "createrepo failed: $?\n"; +} + +if (! -d "$localrepo/scratch") { + mkdir "$localrepo/scratch" + or die "mkdir $localrepo/scratch: $!\nIf you haven't set up a local repository yet, you must read the README file.\n"; +} + +system "rm -f $localrepo/scratch/*"; + +foreach my $name (@buildorder) { + my $version = $srpms{$name}->{version}; + my $release = $srpms{$name}->{release}; + my $srpm_filename = $srpms{$name}->{filename}; + + $release =~ s/\.fc?\d+$//; # "1.fc9" -> "1" + + foreach my $arch (@arches) { + foreach my $distro (@distros) { + # Does the built (binary) package exist already? + my $pattern = "$localrepo/$distro/$arch/RPMS/$name-$version-$release.*.rpm"; + #print "pattern = $pattern\n"; + my @binaries = glob $pattern; + + if (@binaries == 0) + { + # Rebuild the package. + print "*** building $name-$version-$release $arch $distro ***\n"; + + createrepo ($arch, $distro); + system ("mock -r $distro-$arch --resultdir $localrepo/scratch $srpm_filename") == 0 + or die "Build failed, return code $?\nLeaving the logs in $localrepo/scratch\n"; + + # Build was a success so move the final RPMs into the + # mock repo for next time. + system ("mv $localrepo/scratch/*.src.rpm $localrepo/$distro/src/SRPMS") == 0 or die "mv"; + system ("mv $localrepo/scratch/*.rpm $localrepo/$distro/$arch/RPMS") == 0 or die "mv"; + my_mkdir "$localrepo/$distro/$arch/logs/$name-$version-$release"; + system ("mv $localrepo/scratch/*.log $localrepo/$distro/$arch/logs/$name-$version-$release/") == 0 or die "mv"; + + createrepo ($arch, $distro); + } + else + { + print "skipping $name-$version-$release $arch $distro\n"; + } + } + } +} diff --git a/smock/smock.sh b/smock/smock.sh deleted file mode 100755 index 013aa3e..0000000 --- a/smock/smock.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh - -if [ -z "$LOCALREPO" -o -z "$ARCHES" ]; then - echo '$LOCALREPO must point to local repository' - echo '$ARCHES must contain list of architectures to build' - exit 1 -fi - -help() { - echo "syntax: $0 DIST SRPM" -} - -if [ -z "$1" ]; then - help - exit -fi - - -if [ -z "$2" ]; then - help - exit -fi - -DIST=$1 -SRPM=$2 - -createrepos() { - - ( - mkdir -p $LOCALREPO/$DIST/src/SRPMS - cd $LOCALREPO/$DIST/src - rm -rf repodata - createrepo . - ) - - for ARCH in $ARCHES - do - ( - mkdir -p $LOCALREPO/$DIST/$ARCH/RPMS - mkdir -p $LOCALREPO/$DIST/$ARCH/logs - cd $LOCALREPO/$DIST/$ARCH - rm -rf repodata - createrepo --exclude "logs/*rpm" . - ) - done -} - -createrepos - -mkdir -p $LOCALREPO/scratch -rm -f $LOCALREPO/scratch/* - -for ARCH in $ARCHES -do - mkdir -p $LOCALREPO/$DIST/$ARCH/logs/$SRPM - - mock -r $DIST-$ARCH --resultdir $LOCALREPO/scratch $SRPM - - if [ $? != 0 ]; then - echo "Build failed, leaving logs in $LOCALREPO/scratch" - exit 1 - fi - mv $LOCALREPO/scratch/*.src.rpm $LOCALREPO/$DIST/src/SRPMS - mv $LOCALREPO/scratch/*.rpm $LOCALREPO/$DIST/$ARCH/RPMS - mv $LOCALREPO/scratch/*.log $LOCALREPO/$DIST/$ARCH/logs/$SRPM/ -done - -createrepos - -- 1.8.3.1