--- /dev/null
+*~
+*.cmi
+*.cmo
+*.cmx
+*.o
+
+/fedora_ocaml_rebuild
+/libguestfs_upstream
\ No newline at end of file
--- /dev/null
+# Is it ironic that we use a Makefile to build these goals?
+
+# Comment out the next line if goaljobs is installed. If it is not
+# installed, change the path to the location of the goaljobs build
+# directory.
+pkgdir = ../goaljobs
+
+ifeq ($(pkgdir),)
+goaljobs = goaljobs
+else
+goaljobs = $(pkgdir)/goaljobs --pkgdir=$(pkgdir)
+endif
+
+all: \
+ fedora_ocaml_rebuild \
+ libguestfs_upstream
+
+fedora_ocaml_rebuild: config.ml utils.ml fedora.ml fedora_ocaml_rebuild.ml
+ $(goaljobs) --package pcre,extlib $^ -o $@
+
+libguestfs_upstream: config.ml libguestfs_upstream.ml
+ $(goaljobs) $^ -o $@
--- /dev/null
+Public-facing goaljobs that are used to manage libguestfs and other
+projects.
+
+See also:
+
+http://people.redhat.com/~rjones/goaljobs
+http://git.annexia.org/?p=goaljobs.git;a=summary
--- /dev/null
+(* This contains lots of configuration specifics which are
+ * unlikely to be useful unless you are me ...
+ *)
+
+open Goaljobs
+open Printf
+
+(* General. *)
+let buildtmp = sprintf "%s/tmp/builds" (Sys.getenv "HOME")
+
+(* libguestfs *)
+let libguestfs_website_cvs =
+ sprintf "%s/d/redhat/websites/libguestfs" (Sys.getenv "HOME")
+
+let libguestfs_localconfigure source =
+ let configure =
+ match source with
+ `Git -> "./autogen.sh"
+ | `Tarball -> "./configure" in
+ sprintf "\
+#!/bin/bash -
+. localenv
+
+%s \\
+ --prefix /usr \\
+ --disable-static \\
+ --with-default-backend=libvirt \\
+ --enable-gcc-warnings \\
+ --enable-gtk-doc \\
+ --enable-valgrind-daemon \\
+ -C \\
+ \"$@\"
+" configure
+
+let libguestfs_localenv = "\
+# Parallel test runs out of resources starting qemu, unclear why.
+export SKIP_TEST_PARALLEL_MOUNT_LOCAL=1
+
+# Fails under valgrind because cpio subprocess has a memory leak.
+export SKIP_TEST_FILE_ARCHITECTURE_11=1
+"
--- /dev/null
+(* Various useful functions for handling Fedora packages & rebuilds. *)
+
+open ExtString
+open Printf
+
+open Goaljobs
+
+(* Repo dir, etc. *)
+let fedora_dir = Sys.getenv "HOME" // "d/fedora"
+let fedora_repo pkg branch = fedora_dir // pkg // branch
+let fedora_specfile pkg branch =
+ sprintf "%s/%s.spec" (fedora_repo pkg branch) pkg
+
+(* Get the current version of a package. *)
+let fedora_verrel pkg branch =
+ shout "
+ cd %s
+ fedpkg verrel
+ " (fedora_repo pkg branch)
+
+(* Note most of these assume that the package is already cloned
+ * under ~/d/fedora using:
+ *
+ * cd ~/d/fedora
+ * fedpkg clone -B pkgname
+ *)
+
+(* Take a list of Fedora source packages and work out the dependencies
+ * (only within the list of source packages). Returns a list of
+ * (package, [list of deps...])
+ *)
+let dependencies branch source_packages =
+ (* For each source package, get the list of binary packages that it
+ * provides. XXX Not sure if this is totally technically correct, but
+ * it seems to work.
+ *)
+ let bin_to_src =
+ List.concat (
+ List.map (
+ fun pkg ->
+ let provides =
+ shlines "rpmspec -q --provides %s | awk '{print $1}'"
+ (fedora_specfile pkg branch) in
+ List.map (fun bin -> (bin, pkg)) provides
+ ) source_packages
+ ) in
+
+ (* For each package, get the list of build requires that appear
+ * elsewhere in the list of packages.
+ *)
+ let mem dep = List.mem dep source_packages in
+
+ List.map (
+ fun pkg ->
+ let deps =
+ shlines "rpmspec -q --buildrequires %s | awk '{print $1}'"
+ (fedora_specfile pkg branch) in
+ let deps = List.map (
+ fun dep ->
+ try List.assoc dep bin_to_src
+ with Not_found -> "xxx" (* filtered out in next line *)
+ ) deps in
+ let deps = Utils.sort_uniq (List.filter mem deps) in
+ (* eprintf "%s <- %s\n" pkg (String.concat " " deps); *)
+ pkg, deps
+ ) source_packages
+
+let contains_substring substr str =
+ try ignore (String.find str substr); true
+ with Invalid_string -> false
+
+(* Check if a Koji (completed successfully) build exists. *)
+(* Not helped by the fact that the 'koji' tool actively
+ * resists automation: RHBZ#760924.
+ *)
+let koji_build_exists =
+ let state_complete = contains_substring "State: COMPLETE" in
+ let state_other = contains_substring "State: " in
+ let no_such_build = contains_substring "No such build" in
+ fun verrel ->
+ (* Once a build is known to be complete, memoize it. *)
+ let key = sprintf "koji_build_complete_%s" verrel in
+ if memory_exists key then
+ true
+ else (
+ let rec loop () =
+ let out = shout "timeout 120 koji buildinfo %s 2>&1 ||:" verrel in
+ if state_complete out then
+ true
+ else if state_other out then
+ false
+ else if no_such_build out then
+ false
+ else (
+ eprintf "%s\n" out;
+ eprintf "koji_build_exists: unknown output\nretrying ...\n%!";
+ loop ()
+ )
+ in
+ let r = loop () in
+ memory_set key "1";
+ r
+ )
+
+(* Perform a Koji build and wait until it finishes. If it fails,
+ * throw an exception.
+ *)
+let koji_build =
+ let created_task = Pcre.regexp "Created task: (\\d+)" in
+ let name_or_service_not_known =
+ contains_substring "Name or service not known" in
+ let completed_successfully = contains_substring "completed successfully" in
+ let failed = contains_substring "FAILED" in
+ fun pkg branch ->
+ let repodir = fedora_repo pkg branch in
+ let out = shout "cd %s && fedpkg build 2>&1 ||:" repodir in
+ let task_id =
+ try
+ let subs = Pcre.exec ~rex:created_task out in
+ int_of_string (Pcre.get_substring subs 1)
+ with Not_found ->
+ failwith "could not find task ID in fedpkg build output" in
+ let rec loop out =
+ if name_or_service_not_known out then (
+ let out =
+ shout "cd %s && koji watch-task %d 2>&1 ||:" repodir task_id in
+ loop out
+ )
+ else if completed_successfully out then
+ ()
+ else if failed out then (
+ eprintf "%s\n%!" out;
+ failwith "koji build failed"
+ )
+ else
+ failwith (sprintf "koji_build: unknown output: %s" out)
+ in
+ loop out
+
+let koji_wait_repo =
+ let successfully_waited = contains_substring "Successfully waited" in
+ fun target verrel ->
+ let out = shout "koji wait-repo %s --build=%s 2>&1 ||:" target verrel in
+ if successfully_waited out then
+ ()
+ else
+ failwith (sprintf "koji_wait_repo: unknown output: %s" out)
--- /dev/null
+(* Perform a complete Fedora OCaml rebuild, in build order. *)
+
+open Printf
+
+open Goaljobs
+open Config
+open Fedora
+
+let branch = "master"
+let koji_target = "rawhide"
+
+(* The name of the rebuild, and also the magic substring that must
+ * appear in the %changelog when the package has been rebuilt.
+ *)
+let rebuild_name = "OCaml 4.01.0"
+
+(* Packages that have problems or we just don't want to build. *)
+let blocked = [
+ "ocaml-libvirt"; (* RHBZ#1009701 *)
+ "ocaml-lwt"; "ocaml-react"; (* loganjerry is handling *)
+]
+
+(* List of OCaml-related source package names. *)
+let source_packages =
+ let dirs = shlines "cd %s && ls -1d ocaml*" fedora_dir in
+ dirs @ [ "alt-ergo"; "apron"; "brltty"; "coccinelle"; "coq";
+ "cduce"; "frama-c"; "gappalib-coq"; "graphviz"; "hivex";
+ "js-of-ocaml"; "llvm"; "plplot"; "whenjobs"; "why3"; "xen" ]
+
+let source_packages =
+ List.filter (fun pkg -> not (List.mem pkg blocked)) source_packages
+
+(* Dependencies of each package. (pkg, [deps ...]) *)
+let pkg_deps = dependencies branch source_packages
+
+(* Goal: rebuild all packages. *)
+let rec goal all () =
+ List.iter (fun pkg -> require (rebuilt pkg)) source_packages
+
+(* Goal: That 'package' has been rebuilt and exists in Koji. *)
+and rebuilt pkg =
+ let deps = List.assoc pkg pkg_deps in
+ let specfile = fedora_specfile pkg branch in
+
+ (* Note the target must be both of these because the old verrel
+ * could exist as a koji build without it having been part of the
+ * rebuild.
+ *)
+ target (file_contains_string specfile rebuild_name &&
+ koji_build_exists (fedora_verrel pkg branch));
+
+ (* All dependent packages must have been done first. *)
+ List.iter (fun dep -> require (rebuilt dep)) deps;
+
+ (* A local test build must succeed. *)
+ require (local_build_succeeded pkg);
+
+ (* Rebuild the package in Koji. *)
+ koji_build pkg branch;
+
+ (* Wait for the build to appear in Koji repo. Note verrel may change. *)
+ koji_wait_repo koji_target (fedora_verrel pkg branch)
+
+and local_build_succeeded pkg =
+ (* The specfile must have been updated. *)
+ require (specfile_updated pkg);
+
+ let key =
+ sprintf "fedora_ocaml_local_build_%s_%s" pkg (fedora_verrel pkg branch) in
+
+ target (memory_exists key);
+
+ (* Do a local test build to ensure the Koji build will work. *)
+ sh "
+ cd %s
+ sudo yum-builddep %s
+ fedpkg local
+ " (fedora_repo pkg branch)
+ (fedora_specfile pkg branch);
+
+ memory_set key "1"
+
+and specfile_updated pkg =
+ let repodir = fedora_repo pkg branch in
+ let specfile = fedora_specfile pkg branch in
+
+ (* XXX Automate common changes. *)
+ let title = rebuild_name ^ " rebuild." in
+ sh "
+ cd %s
+ git pull --rebase
+ rm -rf x86_64 noarch *.src.rpm
+ rpmdev-bumpspec -c %s %s
+ echo 'Please make further changes as required to the spec file %s.spec'
+ echo '(Press return key)'
+ read
+ emacs -nw %s
+ echo 'OK to commit this change? (press ^C if not)'
+ read
+ fedpkg commit -c
+ echo 'OK to push this change? (press ^C if not)'
+ read
+ fedpkg push
+ " repodir
+ (quote title) specfile
+ pkg
+ specfile
--- /dev/null
+(* This goal script is responsible for:
+ * - testing each new commit to the libguestfs source repo
+ * - checking for new upstream releases of libguestfs
+ * - testing new upstream releases
+ * - packaging them as a source tarball
+ * - testing the source tarball on a variety of systems
+ * - if all the above works, uploading the external website
+ * Note this doesn't build the Fedora releases. See 'libguestfs_fedora.ml'.
+ *)
+
+open Goaljobs
+open Printf
+open Config
+
+let package = "libguestfs"
+
+(* Helper object which stores everything about a version. *)
+type info = {
+ version : string; (* The version as a normal string. *)
+ major : int; (* Broken-out version fields. *)
+ minor : int;
+ release: int;
+ is_stable : bool; (* is a stable version of libguestfs? *)
+ branch : string; (* 'master' or 'stable-1.xx' *)
+ package_version : string; (* package-version *)
+ tarball : string; (* package-version.tar.gz *)
+ urlpath : string; (* download/1.X-(stable|development)/tarball *)
+ url : string; (* full download URL of tarball *)
+}
+
+(* Helper: Fetch latest gnulib into $buildtmp/repos/gnulib
+ * XXX Move to Gnulib module.
+ *)
+let get_gnulib () =
+ sh "
+ cd %s/repos
+ if [ ! -d gnulib ]; then git clone git://git.sv.gnu.org/gnulib.git; fi
+ cd gnulib
+ git checkout --force master
+ git pull
+ " buildtmp
+
+(* Goal: the website has been updated to 'version'. *)
+let rec goal website_updated version =
+ target (url_exists version.url);
+
+ require (tarball_created version);
+ require (tarball_tested version);
+
+ (* We only update the website for the development releases. *)
+ if not version.is_stable then
+ require (website_built version);
+
+ require (website_cvs_checked_in version);
+ require (website_rsync_done version)
+
+(* Goal: website has been rsync'd. *)
+and website_rsync_done version =
+ let key = sprintf "libguestfs_website_rsync_done_%s" version.version in
+ target (memory_exists key);
+
+ sh "
+ cd %s
+ echo NOT RUNNING: . .rsync
+ " libguestfs_website_cvs;
+ memory_set key "1"
+
+(* Goal: Tarball added to CVS and CVS checked in. *)
+and website_cvs_checked_in version =
+ let key = sprintf "libguestfs_website_cvs_checked_in_%s" version.version in
+ target (memory_exists key);
+
+ require (tarball_created version);
+ require (tarball_tested version);
+
+ sh "
+ cd %s
+ cp %s/tarballs/%s %s
+ echo NOT RUNNING: cvs add -kb %s
+ echo NOT RUNNING: cvs ci -m \"Version %s\"
+ " libguestfs_website_cvs
+ buildtmp version.tarball version.urlpath
+ version.urlpath
+ version.version
+
+(* Goal: website (local copy) has been built. *)
+and website_built version =
+ let index_file = sprintf "%s/index.html" libguestfs_website_cvs in
+ target (file_contains_string index_file version.version);
+
+ require (tarball_created version);
+ require (tarball_tested version);
+
+ (* We should only update the website on development releases. *)
+ assert (not version.is_stable);
+
+ sh "
+ tar zxf %s/tarballs/%s
+ cd %s
+ echo %s > localconfigure
+ chmod +x localconfigure
+ echo %s > localenv .
+ ./localconfigure
+ make
+ make website
+ " buildtmp version.tarball
+ version.package_version
+ (quote (libguestfs_localconfigure `Tarball))
+ (quote libguestfs_localenv)
+
+(* Goal: the tarball has passed the required set of tests before
+ * a release is allowed.
+ *)
+and tarball_tested version =
+ let key = sprintf "libguestfs_tarball_tested_%s" version.version in
+ target (memory_exists key);
+
+ require (tarball_created version);
+
+ sh "
+ tar zxf %s/tarballs/%s
+ cd %s
+ echo %s > localconfigure
+ chmod +x localconfigure
+ echo %s > localenv .
+ ./localconfigure
+ make
+ make check-release
+ " buildtmp version.tarball
+ version.package_version
+ (quote (libguestfs_localconfigure `Tarball))
+ (quote libguestfs_localenv)
+
+(* Goal: the tarball has been created from git. *)
+and tarball_created version =
+ let filename = sprintf "%s/tarballs/%s" buildtmp version.tarball in
+ target (file_exists filename);
+
+ let repodir = sprintf "%s/repos/%s-%s" buildtmp package version.branch in
+ require (directory_exists repodir);
+
+ sh "
+ cp -a %s libguestfs
+ cd libguestfs
+ git reset --hard %s
+
+ echo %s > localconfigure
+ chmod +x localconfigure
+ echo %s > localenv .
+
+ ./localconfigure
+ make
+ make dist
+ mv %s %s/tarballs/%s
+ " repodir
+ version.version
+ (quote (libguestfs_localconfigure `Git))
+ (quote libguestfs_localenv)
+ version.tarball buildtmp version.tarball
+
+(* Goal: test a commit. *)
+and commit_tested branch commit =
+ let key = sprintf "libguestfs_commit_tested_%s" commit in
+ target (memory_exists key);
+
+ let repodir = sprintf "%s/repos/%s-%s" buildtmp package branch in
+ require (directory_exists repodir);
+
+ sh "
+ cp -a %s libguestfs
+ cd libguestfs
+ git reset --hard %s
+
+ echo %s > localconfigure
+ chmod +x localconfigure
+ echo %s > localenv
+
+ ./localconfigure
+ make
+ make check-release
+ " repodir
+ commit
+ (quote (libguestfs_localconfigure `Git))
+ (quote libguestfs_localenv);
+
+ memory_set key "1"
+
+(* Helper function to make a full 'info' object from a version
+ * number.
+ *)
+let vernames version =
+ Scanf.sscanf version "%d.%d.%d" (
+ fun major minor release ->
+ let is_stable = minor mod 2 = 0 in
+ let branch =
+ if is_stable then
+ sprintf "stable-%d.%d" major minor
+ else
+ sprintf "master" in
+ let package_version = sprintf "%s-%d.%d.%d" package major minor release in
+ let tarball = sprintf "%s.tar.gz" package_version in
+ let urlpath =
+ if is_stable then
+ sprintf "download/%d.%d-stable/%s" major minor tarball
+ else
+ sprintf "download/%d.%d-development/%s" major minor tarball in
+ let url = "http://libguestfs.org/" ^ urlpath in
+ { version = version;
+ major = major; minor = minor; release = release;
+ is_stable = is_stable;
+ branch = branch;
+ package_version = package_version;
+ tarball = tarball;
+ urlpath = urlpath;
+ url = url }
+ )
+
+(* Helper function to read the latest version in a repo and return
+ * the version.
+ *)
+let git_latest_version branch =
+ let v = shout "
+ cd %s/repos/%s-%s
+ git describe --tags --abbrev=0
+ " buildtmp package (quote branch) in
+ vernames v
+
+(* Get the latest commit. *)
+let git_latest_commit branch =
+ shout "
+ cd %s/repos/%s-%s
+ git rev-parse HEAD
+ " buildtmp package (quote branch)
+
+(* Clone or update a repo to the latest version on a branch, by force.
+ * It is cached in name = $buildtmp/repos/<package>-<branch>
+ *)
+let git_force url branch =
+ sh "
+ cd %s/repos
+ if [ ! -d %s-%s ]; then git clone %s %s-%s; fi
+ cd %s-%s
+ git checkout --force %s
+ git pull
+ # Copy or update gnulib
+ git submodule init
+ git submodule update
+ " buildtmp
+ package (quote branch) (quote url) package (quote branch)
+ package (quote branch)
+ (quote branch)
+
+let () =
+ (* Add a periodic job to check for new git commits and test them. *)
+ every 5 minutes ~name:"new libguestfs commit" (
+ fun () ->
+ git_force "https://github.com/libguestfs/libguestfs.git" "master";
+
+ let commit = git_latest_commit "master" in
+ require (commit_tested "master" commit);
+ );
+
+ (* Periodic job to build new tarballs. *)
+ every 5 minutes ~name:"new libguestfs version" (
+ fun () ->
+ git_force "https://github.com/libguestfs/libguestfs.git" "master";
+
+ let version = git_latest_version "master" in
+ require (website_updated version)
+ )
--- /dev/null
+open Printf
+
+open Goaljobs
+open Config
+
+(* From supermin *)
+let rec uniq ?(cmp = Pervasives.compare) = function
+ | [] -> []
+ | [x] -> [x]
+ | x :: y :: xs when cmp x y = 0 ->
+ uniq ~cmp (x :: xs)
+ | x :: y :: xs ->
+ x :: uniq ~cmp (y :: xs)
+
+let sort_uniq ?(cmp = Pervasives.compare) xs =
+ let xs = List.sort cmp xs in
+ let xs = uniq ~cmp xs in
+ xs
git.annexia.org / goaljobs-goals.git / commitdiff