From 50790d6953d8bc68fd373f1c09949f7bff385050 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Tue, 17 Sep 2013 13:48:17 +0100 Subject: [PATCH 1/1] Add libguestfs upstream build rules. --- .gitignore | 8 ++ Makefile | 22 ++++ README | 7 ++ config.ml | 41 ++++++++ fedora.ml | 147 ++++++++++++++++++++++++++ fedora_ocaml_rebuild.ml | 107 +++++++++++++++++++ libguestfs_upstream.ml | 270 ++++++++++++++++++++++++++++++++++++++++++++++++ utils.ml | 18 ++++ 8 files changed, 620 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README create mode 100644 config.ml create mode 100644 fedora.ml create mode 100644 fedora_ocaml_rebuild.ml create mode 100644 libguestfs_upstream.ml create mode 100644 utils.ml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b209374 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*~ +*.cmi +*.cmo +*.cmx +*.o + +/fedora_ocaml_rebuild +/libguestfs_upstream \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..903d26b --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +# Is it ironic that we use a Makefile to build these goals? + +# Comment out the next line if goaljobs is installed. If it is not +# installed, change the path to the location of the goaljobs build +# directory. +pkgdir = ../goaljobs + +ifeq ($(pkgdir),) +goaljobs = goaljobs +else +goaljobs = $(pkgdir)/goaljobs --pkgdir=$(pkgdir) +endif + +all: \ + fedora_ocaml_rebuild \ + libguestfs_upstream + +fedora_ocaml_rebuild: config.ml utils.ml fedora.ml fedora_ocaml_rebuild.ml + $(goaljobs) --package pcre,extlib $^ -o $@ + +libguestfs_upstream: config.ml libguestfs_upstream.ml + $(goaljobs) $^ -o $@ diff --git a/README b/README new file mode 100644 index 0000000..8682749 --- /dev/null +++ b/README @@ -0,0 +1,7 @@ +Public-facing goaljobs that are used to manage libguestfs and other +projects. + +See also: + +http://people.redhat.com/~rjones/goaljobs +http://git.annexia.org/?p=goaljobs.git;a=summary diff --git a/config.ml b/config.ml new file mode 100644 index 0000000..6db20f5 --- /dev/null +++ b/config.ml @@ -0,0 +1,41 @@ +(* This contains lots of configuration specifics which are + * unlikely to be useful unless you are me ... + *) + +open Goaljobs +open Printf + +(* General. *) +let buildtmp = sprintf "%s/tmp/builds" (Sys.getenv "HOME") + +(* libguestfs *) +let libguestfs_website_cvs = + sprintf "%s/d/redhat/websites/libguestfs" (Sys.getenv "HOME") + +let libguestfs_localconfigure source = + let configure = + match source with + `Git -> "./autogen.sh" + | `Tarball -> "./configure" in + sprintf "\ +#!/bin/bash - +. localenv + +%s \\ + --prefix /usr \\ + --disable-static \\ + --with-default-backend=libvirt \\ + --enable-gcc-warnings \\ + --enable-gtk-doc \\ + --enable-valgrind-daemon \\ + -C \\ + \"$@\" +" configure + +let libguestfs_localenv = "\ +# Parallel test runs out of resources starting qemu, unclear why. +export SKIP_TEST_PARALLEL_MOUNT_LOCAL=1 + +# Fails under valgrind because cpio subprocess has a memory leak. +export SKIP_TEST_FILE_ARCHITECTURE_11=1 +" diff --git a/fedora.ml b/fedora.ml new file mode 100644 index 0000000..82324ff --- /dev/null +++ b/fedora.ml @@ -0,0 +1,147 @@ +(* Various useful functions for handling Fedora packages & rebuilds. *) + +open ExtString +open Printf + +open Goaljobs + +(* Repo dir, etc. *) +let fedora_dir = Sys.getenv "HOME" // "d/fedora" +let fedora_repo pkg branch = fedora_dir // pkg // branch +let fedora_specfile pkg branch = + sprintf "%s/%s.spec" (fedora_repo pkg branch) pkg + +(* Get the current version of a package. *) +let fedora_verrel pkg branch = + shout " + cd %s + fedpkg verrel + " (fedora_repo pkg branch) + +(* Note most of these assume that the package is already cloned + * under ~/d/fedora using: + * + * cd ~/d/fedora + * fedpkg clone -B pkgname + *) + +(* Take a list of Fedora source packages and work out the dependencies + * (only within the list of source packages). Returns a list of + * (package, [list of deps...]) + *) +let dependencies branch source_packages = + (* For each source package, get the list of binary packages that it + * provides. XXX Not sure if this is totally technically correct, but + * it seems to work. + *) + let bin_to_src = + List.concat ( + List.map ( + fun pkg -> + let provides = + shlines "rpmspec -q --provides %s | awk '{print $1}'" + (fedora_specfile pkg branch) in + List.map (fun bin -> (bin, pkg)) provides + ) source_packages + ) in + + (* For each package, get the list of build requires that appear + * elsewhere in the list of packages. + *) + let mem dep = List.mem dep source_packages in + + List.map ( + fun pkg -> + let deps = + shlines "rpmspec -q --buildrequires %s | awk '{print $1}'" + (fedora_specfile pkg branch) in + let deps = List.map ( + fun dep -> + try List.assoc dep bin_to_src + with Not_found -> "xxx" (* filtered out in next line *) + ) deps in + let deps = Utils.sort_uniq (List.filter mem deps) in + (* eprintf "%s <- %s\n" pkg (String.concat " " deps); *) + pkg, deps + ) source_packages + +let contains_substring substr str = + try ignore (String.find str substr); true + with Invalid_string -> false + +(* Check if a Koji (completed successfully) build exists. *) +(* Not helped by the fact that the 'koji' tool actively + * resists automation: RHBZ#760924. + *) +let koji_build_exists = + let state_complete = contains_substring "State: COMPLETE" in + let state_other = contains_substring "State: " in + let no_such_build = contains_substring "No such build" in + fun verrel -> + (* Once a build is known to be complete, memoize it. *) + let key = sprintf "koji_build_complete_%s" verrel in + if memory_exists key then + true + else ( + let rec loop () = + let out = shout "timeout 120 koji buildinfo %s 2>&1 ||:" verrel in + if state_complete out then + true + else if state_other out then + false + else if no_such_build out then + false + else ( + eprintf "%s\n" out; + eprintf "koji_build_exists: unknown output\nretrying ...\n%!"; + loop () + ) + in + let r = loop () in + memory_set key "1"; + r + ) + +(* Perform a Koji build and wait until it finishes. If it fails, + * throw an exception. + *) +let koji_build = + let created_task = Pcre.regexp "Created task: (\\d+)" in + let name_or_service_not_known = + contains_substring "Name or service not known" in + let completed_successfully = contains_substring "completed successfully" in + let failed = contains_substring "FAILED" in + fun pkg branch -> + let repodir = fedora_repo pkg branch in + let out = shout "cd %s && fedpkg build 2>&1 ||:" repodir in + let task_id = + try + let subs = Pcre.exec ~rex:created_task out in + int_of_string (Pcre.get_substring subs 1) + with Not_found -> + failwith "could not find task ID in fedpkg build output" in + let rec loop out = + if name_or_service_not_known out then ( + let out = + shout "cd %s && koji watch-task %d 2>&1 ||:" repodir task_id in + loop out + ) + else if completed_successfully out then + () + else if failed out then ( + eprintf "%s\n%!" out; + failwith "koji build failed" + ) + else + failwith (sprintf "koji_build: unknown output: %s" out) + in + loop out + +let koji_wait_repo = + let successfully_waited = contains_substring "Successfully waited" in + fun target verrel -> + let out = shout "koji wait-repo %s --build=%s 2>&1 ||:" target verrel in + if successfully_waited out then + () + else + failwith (sprintf "koji_wait_repo: unknown output: %s" out) diff --git a/fedora_ocaml_rebuild.ml b/fedora_ocaml_rebuild.ml new file mode 100644 index 0000000..b4445dd --- /dev/null +++ b/fedora_ocaml_rebuild.ml @@ -0,0 +1,107 @@ +(* Perform a complete Fedora OCaml rebuild, in build order. *) + +open Printf + +open Goaljobs +open Config +open Fedora + +let branch = "master" +let koji_target = "rawhide" + +(* The name of the rebuild, and also the magic substring that must + * appear in the %changelog when the package has been rebuilt. + *) +let rebuild_name = "OCaml 4.01.0" + +(* Packages that have problems or we just don't want to build. *) +let blocked = [ + "ocaml-libvirt"; (* RHBZ#1009701 *) + "ocaml-lwt"; "ocaml-react"; (* loganjerry is handling *) +] + +(* List of OCaml-related source package names. *) +let source_packages = + let dirs = shlines "cd %s && ls -1d ocaml*" fedora_dir in + dirs @ [ "alt-ergo"; "apron"; "brltty"; "coccinelle"; "coq"; + "cduce"; "frama-c"; "gappalib-coq"; "graphviz"; "hivex"; + "js-of-ocaml"; "llvm"; "plplot"; "whenjobs"; "why3"; "xen" ] + +let source_packages = + List.filter (fun pkg -> not (List.mem pkg blocked)) source_packages + +(* Dependencies of each package. (pkg, [deps ...]) *) +let pkg_deps = dependencies branch source_packages + +(* Goal: rebuild all packages. *) +let rec goal all () = + List.iter (fun pkg -> require (rebuilt pkg)) source_packages + +(* Goal: That 'package' has been rebuilt and exists in Koji. *) +and rebuilt pkg = + let deps = List.assoc pkg pkg_deps in + let specfile = fedora_specfile pkg branch in + + (* Note the target must be both of these because the old verrel + * could exist as a koji build without it having been part of the + * rebuild. + *) + target (file_contains_string specfile rebuild_name && + koji_build_exists (fedora_verrel pkg branch)); + + (* All dependent packages must have been done first. *) + List.iter (fun dep -> require (rebuilt dep)) deps; + + (* A local test build must succeed. *) + require (local_build_succeeded pkg); + + (* Rebuild the package in Koji. *) + koji_build pkg branch; + + (* Wait for the build to appear in Koji repo. Note verrel may change. *) + koji_wait_repo koji_target (fedora_verrel pkg branch) + +and local_build_succeeded pkg = + (* The specfile must have been updated. *) + require (specfile_updated pkg); + + let key = + sprintf "fedora_ocaml_local_build_%s_%s" pkg (fedora_verrel pkg branch) in + + target (memory_exists key); + + (* Do a local test build to ensure the Koji build will work. *) + sh " + cd %s + sudo yum-builddep %s + fedpkg local + " (fedora_repo pkg branch) + (fedora_specfile pkg branch); + + memory_set key "1" + +and specfile_updated pkg = + let repodir = fedora_repo pkg branch in + let specfile = fedora_specfile pkg branch in + + (* XXX Automate common changes. *) + let title = rebuild_name ^ " rebuild." in + sh " + cd %s + git pull --rebase + rm -rf x86_64 noarch *.src.rpm + rpmdev-bumpspec -c %s %s + echo 'Please make further changes as required to the spec file %s.spec' + echo '(Press return key)' + read + emacs -nw %s + echo 'OK to commit this change? (press ^C if not)' + read + fedpkg commit -c + echo 'OK to push this change? (press ^C if not)' + read + fedpkg push + " repodir + (quote title) specfile + pkg + specfile diff --git a/libguestfs_upstream.ml b/libguestfs_upstream.ml new file mode 100644 index 0000000..1ea64a1 --- /dev/null +++ b/libguestfs_upstream.ml @@ -0,0 +1,270 @@ +(* This goal script is responsible for: + * - testing each new commit to the libguestfs source repo + * - checking for new upstream releases of libguestfs + * - testing new upstream releases + * - packaging them as a source tarball + * - testing the source tarball on a variety of systems + * - if all the above works, uploading the external website + * Note this doesn't build the Fedora releases. See 'libguestfs_fedora.ml'. + *) + +open Goaljobs +open Printf +open Config + +let package = "libguestfs" + +(* Helper object which stores everything about a version. *) +type info = { + version : string; (* The version as a normal string. *) + major : int; (* Broken-out version fields. *) + minor : int; + release: int; + is_stable : bool; (* is a stable version of libguestfs? *) + branch : string; (* 'master' or 'stable-1.xx' *) + package_version : string; (* package-version *) + tarball : string; (* package-version.tar.gz *) + urlpath : string; (* download/1.X-(stable|development)/tarball *) + url : string; (* full download URL of tarball *) +} + +(* Helper: Fetch latest gnulib into $buildtmp/repos/gnulib + * XXX Move to Gnulib module. + *) +let get_gnulib () = + sh " + cd %s/repos + if [ ! -d gnulib ]; then git clone git://git.sv.gnu.org/gnulib.git; fi + cd gnulib + git checkout --force master + git pull + " buildtmp + +(* Goal: the website has been updated to 'version'. *) +let rec goal website_updated version = + target (url_exists version.url); + + require (tarball_created version); + require (tarball_tested version); + + (* We only update the website for the development releases. *) + if not version.is_stable then + require (website_built version); + + require (website_cvs_checked_in version); + require (website_rsync_done version) + +(* Goal: website has been rsync'd. *) +and website_rsync_done version = + let key = sprintf "libguestfs_website_rsync_done_%s" version.version in + target (memory_exists key); + + sh " + cd %s + echo NOT RUNNING: . .rsync + " libguestfs_website_cvs; + memory_set key "1" + +(* Goal: Tarball added to CVS and CVS checked in. *) +and website_cvs_checked_in version = + let key = sprintf "libguestfs_website_cvs_checked_in_%s" version.version in + target (memory_exists key); + + require (tarball_created version); + require (tarball_tested version); + + sh " + cd %s + cp %s/tarballs/%s %s + echo NOT RUNNING: cvs add -kb %s + echo NOT RUNNING: cvs ci -m \"Version %s\" + " libguestfs_website_cvs + buildtmp version.tarball version.urlpath + version.urlpath + version.version + +(* Goal: website (local copy) has been built. *) +and website_built version = + let index_file = sprintf "%s/index.html" libguestfs_website_cvs in + target (file_contains_string index_file version.version); + + require (tarball_created version); + require (tarball_tested version); + + (* We should only update the website on development releases. *) + assert (not version.is_stable); + + sh " + tar zxf %s/tarballs/%s + cd %s + echo %s > localconfigure + chmod +x localconfigure + echo %s > localenv . + ./localconfigure + make + make website + " buildtmp version.tarball + version.package_version + (quote (libguestfs_localconfigure `Tarball)) + (quote libguestfs_localenv) + +(* Goal: the tarball has passed the required set of tests before + * a release is allowed. + *) +and tarball_tested version = + let key = sprintf "libguestfs_tarball_tested_%s" version.version in + target (memory_exists key); + + require (tarball_created version); + + sh " + tar zxf %s/tarballs/%s + cd %s + echo %s > localconfigure + chmod +x localconfigure + echo %s > localenv . + ./localconfigure + make + make check-release + " buildtmp version.tarball + version.package_version + (quote (libguestfs_localconfigure `Tarball)) + (quote libguestfs_localenv) + +(* Goal: the tarball has been created from git. *) +and tarball_created version = + let filename = sprintf "%s/tarballs/%s" buildtmp version.tarball in + target (file_exists filename); + + let repodir = sprintf "%s/repos/%s-%s" buildtmp package version.branch in + require (directory_exists repodir); + + sh " + cp -a %s libguestfs + cd libguestfs + git reset --hard %s + + echo %s > localconfigure + chmod +x localconfigure + echo %s > localenv . + + ./localconfigure + make + make dist + mv %s %s/tarballs/%s + " repodir + version.version + (quote (libguestfs_localconfigure `Git)) + (quote libguestfs_localenv) + version.tarball buildtmp version.tarball + +(* Goal: test a commit. *) +and commit_tested branch commit = + let key = sprintf "libguestfs_commit_tested_%s" commit in + target (memory_exists key); + + let repodir = sprintf "%s/repos/%s-%s" buildtmp package branch in + require (directory_exists repodir); + + sh " + cp -a %s libguestfs + cd libguestfs + git reset --hard %s + + echo %s > localconfigure + chmod +x localconfigure + echo %s > localenv + + ./localconfigure + make + make check-release + " repodir + commit + (quote (libguestfs_localconfigure `Git)) + (quote libguestfs_localenv); + + memory_set key "1" + +(* Helper function to make a full 'info' object from a version + * number. + *) +let vernames version = + Scanf.sscanf version "%d.%d.%d" ( + fun major minor release -> + let is_stable = minor mod 2 = 0 in + let branch = + if is_stable then + sprintf "stable-%d.%d" major minor + else + sprintf "master" in + let package_version = sprintf "%s-%d.%d.%d" package major minor release in + let tarball = sprintf "%s.tar.gz" package_version in + let urlpath = + if is_stable then + sprintf "download/%d.%d-stable/%s" major minor tarball + else + sprintf "download/%d.%d-development/%s" major minor tarball in + let url = "http://libguestfs.org/" ^ urlpath in + { version = version; + major = major; minor = minor; release = release; + is_stable = is_stable; + branch = branch; + package_version = package_version; + tarball = tarball; + urlpath = urlpath; + url = url } + ) + +(* Helper function to read the latest version in a repo and return + * the version. + *) +let git_latest_version branch = + let v = shout " + cd %s/repos/%s-%s + git describe --tags --abbrev=0 + " buildtmp package (quote branch) in + vernames v + +(* Get the latest commit. *) +let git_latest_commit branch = + shout " + cd %s/repos/%s-%s + git rev-parse HEAD + " buildtmp package (quote branch) + +(* Clone or update a repo to the latest version on a branch, by force. + * It is cached in name = $buildtmp/repos/- + *) +let git_force url branch = + sh " + cd %s/repos + if [ ! -d %s-%s ]; then git clone %s %s-%s; fi + cd %s-%s + git checkout --force %s + git pull + # Copy or update gnulib + git submodule init + git submodule update + " buildtmp + package (quote branch) (quote url) package (quote branch) + package (quote branch) + (quote branch) + +let () = + (* Add a periodic job to check for new git commits and test them. *) + every 5 minutes ~name:"new libguestfs commit" ( + fun () -> + git_force "https://github.com/libguestfs/libguestfs.git" "master"; + + let commit = git_latest_commit "master" in + require (commit_tested "master" commit); + ); + + (* Periodic job to build new tarballs. *) + every 5 minutes ~name:"new libguestfs version" ( + fun () -> + git_force "https://github.com/libguestfs/libguestfs.git" "master"; + + let version = git_latest_version "master" in + require (website_updated version) + ) diff --git a/utils.ml b/utils.ml new file mode 100644 index 0000000..25fc740 --- /dev/null +++ b/utils.ml @@ -0,0 +1,18 @@ +open Printf + +open Goaljobs +open Config + +(* From supermin *) +let rec uniq ?(cmp = Pervasives.compare) = function + | [] -> [] + | [x] -> [x] + | x :: y :: xs when cmp x y = 0 -> + uniq ~cmp (x :: xs) + | x :: y :: xs -> + x :: uniq ~cmp (y :: xs) + +let sort_uniq ?(cmp = Pervasives.compare) xs = + let xs = List.sort cmp xs in + let xs = uniq ~cmp xs in + xs -- 1.8.3.1