Add libguestfs upstream build rules.

diff --git a/.gitignore b/.gitignore
new file mode 100644 (file)
index 0000000..b209374
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+*~
+*.cmi
+*.cmo
+*.cmx
+*.o
+
+/fedora_ocaml_rebuild
+/libguestfs_upstream
\ No newline at end of file

diff --git a/Makefile b/Makefile
new file mode 100644 (file)
index 0000000..903d26b
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,22 @@
+# Is it ironic that we use a Makefile to build these goals?
+
+# Comment out the next line if goaljobs is installed.  If it is not
+# installed, change the path to the location of the goaljobs build
+# directory.
+pkgdir = ../goaljobs
+
+ifeq ($(pkgdir),)
+goaljobs = goaljobs
+else
+goaljobs = $(pkgdir)/goaljobs --pkgdir=$(pkgdir)
+endif
+
+all: \
+       fedora_ocaml_rebuild \
+       libguestfs_upstream
+
+fedora_ocaml_rebuild: config.ml utils.ml fedora.ml fedora_ocaml_rebuild.ml
+       $(goaljobs) --package pcre,extlib $^ -o $@
+
+libguestfs_upstream: config.ml libguestfs_upstream.ml
+       $(goaljobs) $^ -o $@

diff --git a/README b/README
new file mode 100644 (file)
index 0000000..8682749
--- /dev/null
+++ b/README
@@ -0,0 +1,7 @@
+Public-facing goaljobs that are used to manage libguestfs and other
+projects.
+
+See also:
+
+http://people.redhat.com/~rjones/goaljobs
+http://git.annexia.org/?p=goaljobs.git;a=summary

diff --git a/config.ml b/config.ml
new file mode 100644 (file)
index 0000000..6db20f5
--- /dev/null
+++ b/config.ml
@@ -0,0 +1,41 @@
+(* This contains lots of configuration specifics which are
+ * unlikely to be useful unless you are me ...
+ *)
+
+open Goaljobs
+open Printf
+
+(* General. *)
+let buildtmp = sprintf "%s/tmp/builds" (Sys.getenv "HOME")
+
+(* libguestfs *)
+let libguestfs_website_cvs =
+  sprintf "%s/d/redhat/websites/libguestfs" (Sys.getenv "HOME")
+
+let libguestfs_localconfigure source =
+  let configure =
+    match source with
+      `Git -> "./autogen.sh"
+    | `Tarball -> "./configure" in
+  sprintf "\
+#!/bin/bash -
+. localenv
+
+%s \\
+  --prefix /usr \\
+  --disable-static \\
+  --with-default-backend=libvirt \\
+  --enable-gcc-warnings \\
+  --enable-gtk-doc \\
+  --enable-valgrind-daemon \\
+  -C \\
+  \"$@\"
+" configure
+
+let libguestfs_localenv = "\
+# Parallel test runs out of resources starting qemu, unclear why.
+export SKIP_TEST_PARALLEL_MOUNT_LOCAL=1
+
+# Fails under valgrind because cpio subprocess has a memory leak.
+export SKIP_TEST_FILE_ARCHITECTURE_11=1
+"

diff --git a/fedora.ml b/fedora.ml
new file mode 100644 (file)
index 0000000..82324ff
--- /dev/null
+++ b/fedora.ml
@@ -0,0 +1,147 @@
+(* Various useful functions for handling Fedora packages & rebuilds. *)
+
+open ExtString
+open Printf
+
+open Goaljobs
+
+(* Repo dir, etc. *)
+let fedora_dir = Sys.getenv "HOME" // "d/fedora"
+let fedora_repo pkg branch = fedora_dir // pkg // branch
+let fedora_specfile pkg branch =
+  sprintf "%s/%s.spec" (fedora_repo pkg branch) pkg
+
+(* Get the current version of a package. *)
+let fedora_verrel pkg branch =
+  shout "
+    cd %s
+    fedpkg verrel
+  " (fedora_repo pkg branch)
+
+(* Note most of these assume that the package is already cloned
+ * under ~/d/fedora using:
+ *
+ * cd ~/d/fedora
+ * fedpkg clone -B pkgname
+ *)
+
+(* Take a list of Fedora source packages and work out the dependencies
+ * (only within the list of source packages).  Returns a list of
+ * (package, [list of deps...])
+ *)
+let dependencies branch source_packages =
+  (* For each source package, get the list of binary packages that it
+   * provides.  XXX Not sure if this is totally technically correct, but
+   * it seems to work.
+   *)
+  let bin_to_src =
+    List.concat (
+      List.map (
+        fun pkg ->
+          let provides =
+            shlines "rpmspec -q --provides %s | awk '{print $1}'"
+              (fedora_specfile pkg branch) in
+          List.map (fun bin -> (bin, pkg)) provides
+      ) source_packages
+    ) in
+
+  (* For each package, get the list of build requires that appear
+   * elsewhere in the list of packages.
+   *)
+  let mem dep = List.mem dep source_packages in
+
+  List.map (
+    fun pkg ->
+      let deps =
+        shlines "rpmspec -q --buildrequires %s | awk '{print $1}'"
+          (fedora_specfile pkg branch) in
+      let deps = List.map (
+        fun dep ->
+          try List.assoc dep bin_to_src
+          with Not_found -> "xxx" (* filtered out in next line *)
+      ) deps in
+      let deps = Utils.sort_uniq (List.filter mem deps) in
+      (* eprintf "%s <- %s\n" pkg (String.concat " " deps); *)
+      pkg, deps
+  ) source_packages
+
+let contains_substring substr str =
+  try ignore (String.find str substr); true
+  with Invalid_string -> false
+
+(* Check if a Koji (completed successfully) build exists. *)
+(* Not helped by the fact that the 'koji' tool actively
+ * resists automation: RHBZ#760924.
+ *)
+let koji_build_exists =
+  let state_complete = contains_substring "State: COMPLETE" in
+  let state_other = contains_substring "State: " in
+  let no_such_build = contains_substring "No such build" in
+  fun verrel ->
+    (* Once a build is known to be complete, memoize it. *)
+    let key = sprintf "koji_build_complete_%s" verrel in
+    if memory_exists key then
+      true
+    else (
+      let rec loop () =
+        let out = shout "timeout 120 koji buildinfo %s 2>&1 ||:" verrel in
+        if state_complete out then
+          true
+        else if state_other out then
+          false
+        else if no_such_build out then
+          false
+        else (
+          eprintf "%s\n" out;
+          eprintf "koji_build_exists: unknown output\nretrying ...\n%!";
+          loop ()
+        )
+      in
+      let r = loop () in
+      memory_set key "1";
+      r
+    )
+
+(* Perform a Koji build and wait until it finishes.  If it fails,
+ * throw an exception.
+ *)
+let koji_build =
+  let created_task = Pcre.regexp "Created task: (\\d+)" in
+  let name_or_service_not_known =
+    contains_substring "Name or service not known" in
+  let completed_successfully = contains_substring "completed successfully" in
+  let failed = contains_substring "FAILED" in
+  fun pkg branch ->
+    let repodir = fedora_repo pkg branch in
+    let out = shout "cd %s && fedpkg build 2>&1 ||:" repodir in
+    let task_id =
+      try
+        let subs = Pcre.exec ~rex:created_task out in
+        int_of_string (Pcre.get_substring subs 1)
+      with Not_found ->
+        failwith "could not find task ID in fedpkg build output" in
+    let rec loop out =
+      if name_or_service_not_known out then (
+        let out =
+          shout "cd %s && koji watch-task %d 2>&1 ||:" repodir task_id in
+        loop out
+      )
+      else if completed_successfully out then
+        ()
+      else if failed out then (
+        eprintf "%s\n%!" out;
+        failwith "koji build failed"
+      )
+      else
+        failwith (sprintf "koji_build: unknown output: %s" out)
+    in
+    loop out
+
+let koji_wait_repo =
+  let successfully_waited = contains_substring "Successfully waited" in
+  fun target verrel ->
+    let out = shout "koji wait-repo %s --build=%s 2>&1 ||:" target verrel in
+    if successfully_waited out then
+      ()
+    else
+      failwith (sprintf "koji_wait_repo: unknown output: %s" out)

diff --git a/fedora_ocaml_rebuild.ml b/fedora_ocaml_rebuild.ml
new file mode 100644 (file)
index 0000000..b4445dd
--- /dev/null
+++ b/fedora_ocaml_rebuild.ml
@@ -0,0 +1,107 @@
+(* Perform a complete Fedora OCaml rebuild, in build order. *)
+
+open Printf
+
+open Goaljobs
+open Config
+open Fedora
+
+let branch = "master"
+let koji_target = "rawhide"
+
+(* The name of the rebuild, and also the magic substring that must
+ * appear in the %changelog when the package has been rebuilt.
+ *)
+let rebuild_name = "OCaml 4.01.0"
+
+(* Packages that have problems or we just don't want to build. *)
+let blocked = [
+  "ocaml-libvirt"; (* RHBZ#1009701 *)
+  "ocaml-lwt"; "ocaml-react"; (* loganjerry is handling *)
+]
+
+(* List of OCaml-related source package names. *)
+let source_packages =
+  let dirs = shlines "cd %s && ls -1d ocaml*" fedora_dir in
+  dirs @ [ "alt-ergo"; "apron"; "brltty"; "coccinelle"; "coq";
+           "cduce"; "frama-c"; "gappalib-coq"; "graphviz"; "hivex";
+           "js-of-ocaml"; "llvm"; "plplot"; "whenjobs"; "why3"; "xen" ]
+
+let source_packages =
+  List.filter (fun pkg -> not (List.mem pkg blocked)) source_packages
+
+(* Dependencies of each package.  (pkg, [deps ...]) *)
+let pkg_deps = dependencies branch source_packages
+
+(* Goal: rebuild all packages. *)
+let rec goal all () =
+  List.iter (fun pkg -> require (rebuilt pkg)) source_packages
+
+(* Goal: That 'package' has been rebuilt and exists in Koji. *)
+and rebuilt pkg =
+  let deps = List.assoc pkg pkg_deps in
+  let specfile = fedora_specfile pkg branch in
+
+  (* Note the target must be both of these because the old verrel
+   * could exist as a koji build without it having been part of the
+   * rebuild.
+   *)
+  target (file_contains_string specfile rebuild_name &&
+            koji_build_exists (fedora_verrel pkg branch));
+
+  (* All dependent packages must have been done first. *)
+  List.iter (fun dep -> require (rebuilt dep)) deps;
+
+  (* A local test build must succeed. *)
+  require (local_build_succeeded pkg);
+
+  (* Rebuild the package in Koji. *)
+  koji_build pkg branch;
+
+  (* Wait for the build to appear in Koji repo.  Note verrel may change. *)
+  koji_wait_repo koji_target (fedora_verrel pkg branch)
+
+and local_build_succeeded pkg =
+  (* The specfile must have been updated. *)
+  require (specfile_updated pkg);
+
+  let key =
+    sprintf "fedora_ocaml_local_build_%s_%s" pkg (fedora_verrel pkg branch) in
+
+  target (memory_exists key);
+
+  (* Do a local test build to ensure the Koji build will work. *)
+  sh "
+    cd %s
+    sudo yum-builddep %s
+    fedpkg local
+  " (fedora_repo pkg branch)
+    (fedora_specfile pkg branch);
+
+  memory_set key "1"
+
+and specfile_updated pkg =
+  let repodir = fedora_repo pkg branch in
+  let specfile = fedora_specfile pkg branch in
+
+  (* XXX Automate common changes. *)
+  let title = rebuild_name ^ " rebuild." in
+  sh "
+    cd %s
+    git pull --rebase
+    rm -rf x86_64 noarch *.src.rpm
+    rpmdev-bumpspec -c %s %s
+    echo 'Please make further changes as required to the spec file %s.spec'
+    echo '(Press return key)'
+    read
+    emacs -nw %s
+    echo 'OK to commit this change? (press ^C if not)'
+    read
+    fedpkg commit -c
+    echo 'OK to push this change? (press ^C if not)'
+    read
+    fedpkg push
+  " repodir
+    (quote title) specfile
+    pkg
+    specfile 

diff --git a/libguestfs_upstream.ml b/libguestfs_upstream.ml
new file mode 100644 (file)
index 0000000..1ea64a1
--- /dev/null
+++ b/libguestfs_upstream.ml
@@ -0,0 +1,270 @@
+(* This goal script is responsible for:
+ *  - testing each new commit to the libguestfs source repo
+ *  - checking for new upstream releases of libguestfs
+ *  - testing new upstream releases
+ *  - packaging them as a source tarball
+ *  - testing the source tarball on a variety of systems
+ *  - if all the above works, uploading the external website
+ * Note this doesn't build the Fedora releases.  See 'libguestfs_fedora.ml'.
+ *)
+
+open Goaljobs
+open Printf
+open Config
+
+let package = "libguestfs"
+
+(* Helper object which stores everything about a version. *)
+type info = {
+  version : string;              (* The version as a normal string. *)
+  major : int;                   (* Broken-out version fields. *)
+  minor : int;
+  release: int;
+  is_stable : bool;              (* is a stable version of libguestfs? *)
+  branch : string;               (* 'master' or 'stable-1.xx' *)
+  package_version : string;      (* package-version *)
+  tarball : string;              (* package-version.tar.gz *)
+  urlpath : string;              (* download/1.X-(stable|development)/tarball *)
+  url : string;                  (* full download URL of tarball *)
+}
+
+(* Helper: Fetch latest gnulib into $buildtmp/repos/gnulib
+ * XXX Move to Gnulib module.
+ *)
+let get_gnulib () =
+  sh "
+    cd %s/repos
+    if [ ! -d gnulib ]; then git clone git://git.sv.gnu.org/gnulib.git; fi
+    cd gnulib
+    git checkout --force master
+    git pull
+  " buildtmp
+
+(* Goal: the website has been updated to 'version'. *)
+let rec goal website_updated version =
+  target (url_exists version.url);
+
+  require (tarball_created version);
+  require (tarball_tested version);
+
+  (* We only update the website for the development releases. *)
+  if not version.is_stable then
+    require (website_built version);
+
+  require (website_cvs_checked_in version);
+  require (website_rsync_done version)
+
+(* Goal: website has been rsync'd. *)
+and website_rsync_done version =
+  let key = sprintf "libguestfs_website_rsync_done_%s" version.version in
+  target (memory_exists key);
+
+  sh "
+    cd %s
+    echo NOT RUNNING: . .rsync
+  " libguestfs_website_cvs;
+  memory_set key "1"
+
+(* Goal: Tarball added to CVS and CVS checked in. *)
+and website_cvs_checked_in version =
+  let key = sprintf "libguestfs_website_cvs_checked_in_%s" version.version in
+  target (memory_exists key);
+
+  require (tarball_created version);
+  require (tarball_tested version);
+
+  sh "
+    cd %s
+    cp %s/tarballs/%s %s
+    echo NOT RUNNING: cvs add -kb %s
+    echo NOT RUNNING: cvs ci -m \"Version %s\"
+  " libguestfs_website_cvs
+    buildtmp version.tarball version.urlpath
+    version.urlpath
+    version.version
+
+(* Goal: website (local copy) has been built. *)
+and website_built version =
+  let index_file = sprintf "%s/index.html" libguestfs_website_cvs in
+  target (file_contains_string index_file version.version);
+
+  require (tarball_created version);
+  require (tarball_tested version);
+
+  (* We should only update the website on development releases. *)
+  assert (not version.is_stable);
+
+  sh "
+    tar zxf %s/tarballs/%s
+    cd %s
+    echo %s > localconfigure
+    chmod +x localconfigure
+    echo %s > localenv .
+    ./localconfigure
+    make
+    make website
+  " buildtmp version.tarball
+    version.package_version
+    (quote (libguestfs_localconfigure `Tarball))
+    (quote libguestfs_localenv)
+
+(* Goal: the tarball has passed the required set of tests before
+ * a release is allowed.
+ *)
+and tarball_tested version =
+  let key = sprintf "libguestfs_tarball_tested_%s" version.version in
+  target (memory_exists key);
+
+  require (tarball_created version);
+
+  sh "
+    tar zxf %s/tarballs/%s
+    cd %s
+    echo %s > localconfigure
+    chmod +x localconfigure
+    echo %s > localenv .
+    ./localconfigure
+    make
+    make check-release
+  " buildtmp version.tarball
+    version.package_version
+    (quote (libguestfs_localconfigure `Tarball))
+    (quote libguestfs_localenv)
+
+(* Goal: the tarball has been created from git. *)
+and tarball_created version =
+  let filename = sprintf "%s/tarballs/%s" buildtmp version.tarball in
+  target (file_exists filename);
+
+  let repodir = sprintf "%s/repos/%s-%s" buildtmp package version.branch in
+  require (directory_exists repodir);
+
+  sh "
+    cp -a %s libguestfs
+    cd libguestfs
+    git reset --hard %s
+
+    echo %s > localconfigure
+    chmod +x localconfigure
+    echo %s > localenv .
+
+    ./localconfigure
+    make
+    make dist
+    mv %s %s/tarballs/%s
+  " repodir
+    version.version
+    (quote (libguestfs_localconfigure `Git))
+    (quote libguestfs_localenv)
+    version.tarball buildtmp version.tarball
+
+(* Goal: test a commit. *)
+and commit_tested branch commit =
+  let key = sprintf "libguestfs_commit_tested_%s" commit in
+  target (memory_exists key);
+
+  let repodir = sprintf "%s/repos/%s-%s" buildtmp package branch in
+  require (directory_exists repodir);
+
+  sh "
+    cp -a %s libguestfs
+    cd libguestfs
+    git reset --hard %s
+
+    echo %s > localconfigure
+    chmod +x localconfigure
+    echo %s > localenv
+
+    ./localconfigure
+    make
+    make check-release
+  " repodir
+    commit
+    (quote (libguestfs_localconfigure `Git))
+    (quote libguestfs_localenv);
+
+  memory_set key "1"
+
+(* Helper function to make a full 'info' object from a version
+ * number.
+ *)
+let vernames version =
+  Scanf.sscanf version "%d.%d.%d" (
+    fun major minor release ->
+      let is_stable = minor mod 2 = 0 in
+      let branch =
+        if is_stable then
+          sprintf "stable-%d.%d" major minor
+        else
+          sprintf "master" in
+      let package_version = sprintf "%s-%d.%d.%d" package major minor release in
+      let tarball = sprintf "%s.tar.gz" package_version in
+      let urlpath =
+        if is_stable then
+          sprintf "download/%d.%d-stable/%s" major minor tarball
+        else
+          sprintf "download/%d.%d-development/%s" major minor tarball in
+      let url = "http://libguestfs.org/" ^ urlpath in
+      { version = version;
+        major = major; minor = minor; release = release;
+        is_stable = is_stable;
+        branch = branch;
+        package_version = package_version;
+        tarball = tarball;
+        urlpath = urlpath;
+        url = url }
+  )
+
+(* Helper function to read the latest version in a repo and return
+ * the version.
+ *)
+let git_latest_version branch =
+  let v = shout "
+    cd %s/repos/%s-%s
+    git describe --tags --abbrev=0
+  " buildtmp package (quote branch) in
+  vernames v
+
+(* Get the latest commit. *)
+let git_latest_commit branch =
+  shout "
+    cd %s/repos/%s-%s
+    git rev-parse HEAD
+  " buildtmp package (quote branch)
+
+(* Clone or update a repo to the latest version on a branch, by force.
+ * It is cached in name = $buildtmp/repos/<package>-<branch>
+ *)
+let git_force url branch =
+  sh "
+    cd %s/repos
+    if [ ! -d %s-%s ]; then git clone %s %s-%s; fi
+    cd %s-%s
+    git checkout --force %s
+    git pull
+    # Copy or update gnulib
+    git submodule init
+    git submodule update
+  " buildtmp
+    package (quote branch) (quote url) package (quote branch)
+    package (quote branch)
+    (quote branch)
+
+let () =
+  (* Add a periodic job to check for new git commits and test them. *)
+  every 5 minutes ~name:"new libguestfs commit" (
+    fun () ->
+      git_force "https://github.com/libguestfs/libguestfs.git" "master";
+
+      let commit = git_latest_commit "master" in
+      require (commit_tested "master" commit);
+  );
+
+  (* Periodic job to build new tarballs. *)
+  every 5 minutes ~name:"new libguestfs version" (
+    fun () ->
+      git_force "https://github.com/libguestfs/libguestfs.git" "master";
+
+      let version = git_latest_version "master" in
+      require (website_updated version)
+  )

diff --git a/utils.ml b/utils.ml
new file mode 100644 (file)
index 0000000..25fc740
--- /dev/null
+++ b/utils.ml
@@ -0,0 +1,18 @@
+open Printf
+
+open Goaljobs
+open Config
+
+(* From supermin *)
+let rec uniq ?(cmp = Pervasives.compare) = function
+  | [] -> []
+  | [x] -> [x]
+  | x :: y :: xs when cmp x y = 0 ->
+      uniq ~cmp (x :: xs)
+  | x :: y :: xs ->
+      x :: uniq ~cmp (y :: xs)
+
+let sort_uniq ?(cmp = Pervasives.compare) xs =
+  let xs = List.sort cmp xs in
+  let xs = uniq ~cmp xs in
+  xs