From: Richard W.M. Jones Date: Thu, 3 Oct 2013 17:43:07 +0000 (+0100) Subject: Add libguestfs_fedora script to manage Fedora builds. X-Git-Url: http://git.annexia.org/?a=commitdiff_plain;h=8e4d36a8ec04de7ec2ae22f336708adb537a7e59;p=goaljobs-goals.git Add libguestfs_fedora script to manage Fedora builds. --- diff --git a/.gitignore b/.gitignore index b209374..38779fd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ *.o /fedora_ocaml_rebuild +/libguestfs_fedora /libguestfs_upstream \ No newline at end of file diff --git a/Makefile b/Makefile index 55fdc67..c927bda 100644 --- a/Makefile +++ b/Makefile @@ -6,21 +6,25 @@ pkgdir = ../goaljobs ifeq ($(pkgdir),) -goaljobs = goaljobs +goaljobs = goaljobs --package pcre,extlib else -goaljobs = $(pkgdir)/goaljobs --pkgdir=$(pkgdir) +goaljobs = $(pkgdir)/goaljobs --pkgdir=$(pkgdir) --package pcre,extlib endif programs = \ fedora_ocaml_rebuild \ - libguestfs_upstream + libguestfs_upstream \ + libguestfs_fedora all: $(programs) fedora_ocaml_rebuild: config.ml utils.ml git.ml fedora.ml fedora_ocaml_rebuild.ml - $(goaljobs) --package pcre,extlib $^ -o $@ + $(goaljobs) $^ -o $@ + +libguestfs_upstream: config.ml utils.ml libguestfs.ml libguestfs_upstream.ml + $(goaljobs) $^ -o $@ -libguestfs_upstream: config.ml libguestfs_upstream.ml +libguestfs_fedora: config.ml utils.ml git.ml fedora.ml libguestfs.ml libguestfs_fedora.ml $(goaljobs) $^ -o $@ clean: diff --git a/libguestfs.ml b/libguestfs.ml new file mode 100644 index 0000000..e265608 --- /dev/null +++ b/libguestfs.ml @@ -0,0 +1,118 @@ +(* Helper functions for handling libguestfs. *) + +open Goaljobs + +open Printf + +open Config +open Utils + +let package = "libguestfs" + +(* Helper object which stores everything about a version. *) +type info = { + version : string; (* The version as a normal string. *) + major : int; (* Broken-out version fields. *) + minor : int; + release: int; + is_stable : bool; (* is a stable version of libguestfs? *) + branch : string; (* 'master' or 'stable-1.xx' *) + package_version : string; (* package-version *) + tarball : string; (* package-version.tar.gz *) + urlpath : string; (* download/1.X-(stable|development)/tarball *) + url : string; (* full download URL of tarball *) +} + +let compare_versions { major = major1; minor = minor1; release = release1 } + { major = major2; minor = minor2; release = release2 } = + compare (major1, minor1, release1) (major2, minor2, release2) + +let string_of_libguestfs_version { version = version } = version + +(* Helper function to make a full 'info' object from a version + * number. + *) +let vernames version = + Scanf.sscanf version "%d.%d.%d" ( + fun major minor release -> + let is_stable = minor mod 2 = 0 in + let branch = + if is_stable then + sprintf "stable-%d.%d" major minor + else + sprintf "master" in + let package_version = sprintf "%s-%d.%d.%d" package major minor release in + let tarball = sprintf "%s.tar.gz" package_version in + let urlpath = + if is_stable then + sprintf "download/%d.%d-stable/%s" major minor tarball + else + sprintf "download/%d.%d-development/%s" major minor tarball in + let url = "http://libguestfs.org/" ^ urlpath in + { version = version; + major = major; minor = minor; release = release; + is_stable = is_stable; + branch = branch; + package_version = package_version; + tarball = tarball; + urlpath = urlpath; + url = url } + ) + +(* Clone or update a repo to the latest version on a branch, by force. + * It is cached in name = $buildtmp/repos/- + *) +let git_force branch = + let url = "https://github.com/libguestfs/libguestfs.git" in + sh " + cd %s/repos + if [ ! -d %s-%s ]; then git clone %s %s-%s; fi + cd %s-%s + git checkout --force %s + git pull + # Copy or update gnulib + git submodule init + git submodule update + " buildtmp + package (quote branch) (quote url) package (quote branch) + package (quote branch) + (quote branch) + +(* Helper function to read the latest version in a repo and return + * the version. + *) +let git_latest_version branch = + let v = shout " + cd %s/repos/%s-%s + git describe --tags --abbrev=0 + " buildtmp package (quote branch) in + vernames v + +(* Get the latest commit. *) +let git_latest_commit branch = + shout " + cd %s/repos/%s-%s + git rev-parse HEAD + " buildtmp package (quote branch) + +(* Find the latest website tarball version. Actually we use our local + * CVS copy of the website rather than downloading from + * http://libguestfs.org + *) +let website_latest_version = + let rex = Str.regexp "libguestfs-\\(.*\\)\\.tar\\.gz" in + fun branch -> + let wdir = libguestfs_website_cvs // "download" // branch in + let files = Sys.readdir wdir in + let files = Array.to_list files in + if files = [] then None + else ( + let versions = filter_map ( + fun name -> + if not (Str.string_match rex name 0) then None + else Some (vernames (Str.matched_group 1 name)) + ) files in + let versions = List.sort compare_versions versions in + let versions = List.rev versions in + Some (List.hd versions) + ) diff --git a/libguestfs_fedora.ml b/libguestfs_fedora.ml new file mode 100644 index 0000000..18777a8 --- /dev/null +++ b/libguestfs_fedora.ml @@ -0,0 +1,131 @@ +(* Handle Fedora builds of libguestfs. *) + +open Goaljobs + +open Printf + +open Config +open Fedora +open Git +open Libguestfs + +(* Enable debugging. *) +let () = + Unix.putenv "LIBGUESTFS_DEBUG" "1"; + Unix.putenv "LIBGUESTFS_TRACE" "1" + +(* Log program output. *) +let from = "rjones@redhat.com" +let to_ = "rjones@redhat.com" + +let package = "libguestfs" + +(* How branches in libguestfs upstream map to branches in Fedora. *) +let branches = [ + "1.23-development", "master"; (* Rawhide follows development. *) + "1.23-development", "f20"; (* F20 also follows development. *) + "1.22-stable", "f19"; (* F19 follows 1.22. *) + "1.20-stable", "f18"; (* F18 follows 1.20. *) +] + +(* Goal: Latest website version has been built in every branch. *) +let rec goal all () = + List.iter ( + fun (wbranch, fbranch) -> + match website_latest_version wbranch with + | None -> () + | Some version -> + require (fedora_built version fbranch) + ) branches + +(* Goal: Fedora has a successful build of 'version' on 'branch'. *) +and fedora_built version branch = + let specfile = fedora_specfile package branch in + + target (file_contains_string specfile version.version && + koji_build_state (fedora_verrel package branch) == `Complete); + + require (sources_uploaded version branch); + require (specfile_pushed version branch); + + koji_build ~wait:true package branch + +and sources_uploaded version branch = + let repodir = fedora_repo package branch in + let sources = repodir // "sources" in + let key = sprintf "libguestfs_fedora_sources_uploaded_%s" version.version in + + target (file_contains_string sources version.version && + memory_exists key); + onrun (fun () -> memory_set key "1"); + + require (repodir_up_to_date repodir); + + sh " + cd %s + fedpkg new-sources %s/%s + " repodir libguestfs_website_cvs version.urlpath + +and specfile_updated version branch = + let repodir = fedora_repo package branch in + let specfile = fedora_specfile package branch in + + target (file_contains_string specfile version.version); + + require (repodir_up_to_date repodir); + + (* Hairy specfile editing. *) + sh " + cd %s + email=\"Richard W.M. Jones \" + date=`date +\"%%a %%b %%d %%Y\"` + cp libguestfs.spec libguestfs.spec.old + sed < libguestfs.spec.old \\ + -e \"s/^Version:.*/Version: %s/\" \\ + -e \"s/^Release:.*/Release: 1%%{?dist}/\" \\ + -e \"/^%%changelog/a \\ +* $date $email - 1:%s-1\\\\n\\ +- New upstream version %s.\\\\n\\ +\" > libguestfs.spec + rm libguestfs.spec.old + " repodir version.version version.version version.version + +and specfile_committed version branch = + let repodir = fedora_repo package branch in + let key = sprintf "libguestfs_fedora_specfile_committed_%s" version.version in + + target (memory_exists key); + onrun (fun () -> memory_set key "1"); + + require (specfile_updated version branch); + + sh " + cd %s + fedpkg commit -c + " repodir + +and specfile_pushed version branch = + let repodir = fedora_repo package branch in + let key = sprintf "libguestfs_fedora_specfile_pushed_%s" version.version in + + target (memory_exists key); + onrun (fun () -> memory_set key "1"); + + require (repodir_up_to_date repodir); + require (specfile_committed version branch); + + sh " + cd %s + fedpkg push + " repodir + +and repodir_up_to_date repodir = + sh " + cd %s + git fetch + " repodir; + if not (git_has_local_changes repodir) then + sh " + cd %s + git pull --rebase + " repodir diff --git a/libguestfs_upstream.ml b/libguestfs_upstream.ml index 4c32dfd..a3d558f 100644 --- a/libguestfs_upstream.ml +++ b/libguestfs_upstream.ml @@ -11,6 +11,7 @@ open Goaljobs open Printf open Config +open Libguestfs (* Enable debugging. *) let () = @@ -20,37 +21,13 @@ let () = (* Log program output. *) let from = "rjones@redhat.com" let to_ = "rjones@redhat.com" +(* let logfile = log_program_output () let () = eprintf "logging to %s\n%!" logfile +*) let package = "libguestfs" -(* Helper object which stores everything about a version. *) -type info = { - version : string; (* The version as a normal string. *) - major : int; (* Broken-out version fields. *) - minor : int; - release: int; - is_stable : bool; (* is a stable version of libguestfs? *) - branch : string; (* 'master' or 'stable-1.xx' *) - package_version : string; (* package-version *) - tarball : string; (* package-version.tar.gz *) - urlpath : string; (* download/1.X-(stable|development)/tarball *) - url : string; (* full download URL of tarball *) -} - -(* Helper: Fetch latest gnulib into $buildtmp/repos/gnulib - * XXX Move to Gnulib module. - *) -let get_gnulib () = - sh " - cd %s/repos - if [ ! -d gnulib ]; then git clone git://git.sv.gnu.org/gnulib.git; fi - cd gnulib - git checkout --force master - git pull - " buildtmp - (* Goal: the website has been updated to 'version'. *) let rec goal website_updated version = target (url_exists version.url); @@ -179,7 +156,7 @@ and commit_tested branch commit = onfail ( fun _ -> let subject = sprintf "goal: %s: FAILED" goalname in - mailto ~from ~subject ~attach:[logfile] to_ + mailto ~from ~subject (*~attach:[logfile]*) to_ ); let key = sprintf "libguestfs_commit_tested_%s" commit in @@ -206,77 +183,11 @@ and commit_tested branch commit = (quote (libguestfs_localconfigure `Git)) (quote libguestfs_localenv) -(* Helper function to make a full 'info' object from a version - * number. - *) -let vernames version = - Scanf.sscanf version "%d.%d.%d" ( - fun major minor release -> - let is_stable = minor mod 2 = 0 in - let branch = - if is_stable then - sprintf "stable-%d.%d" major minor - else - sprintf "master" in - let package_version = sprintf "%s-%d.%d.%d" package major minor release in - let tarball = sprintf "%s.tar.gz" package_version in - let urlpath = - if is_stable then - sprintf "download/%d.%d-stable/%s" major minor tarball - else - sprintf "download/%d.%d-development/%s" major minor tarball in - let url = "http://libguestfs.org/" ^ urlpath in - { version = version; - major = major; minor = minor; release = release; - is_stable = is_stable; - branch = branch; - package_version = package_version; - tarball = tarball; - urlpath = urlpath; - url = url } - ) - -(* Helper function to read the latest version in a repo and return - * the version. - *) -let git_latest_version branch = - let v = shout " - cd %s/repos/%s-%s - git describe --tags --abbrev=0 - " buildtmp package (quote branch) in - vernames v - -(* Get the latest commit. *) -let git_latest_commit branch = - shout " - cd %s/repos/%s-%s - git rev-parse HEAD - " buildtmp package (quote branch) - -(* Clone or update a repo to the latest version on a branch, by force. - * It is cached in name = $buildtmp/repos/- - *) -let git_force url branch = - sh " - cd %s/repos - if [ ! -d %s-%s ]; then git clone %s %s-%s; fi - cd %s-%s - git checkout --force %s - git pull - # Copy or update gnulib - git submodule init - git submodule update - " buildtmp - package (quote branch) (quote url) package (quote branch) - package (quote branch) - (quote branch) - let () = (* Add a periodic job to check for new git commits and test them. *) every libguestfs_query_mins minutes ~name:"new libguestfs commit" ( fun () -> - git_force "https://github.com/libguestfs/libguestfs.git" "master"; - + git_force "master"; let commit = git_latest_commit "master" in require (commit_tested "master" commit); ); @@ -284,8 +195,7 @@ let () = (* Periodic job to build new tarballs. *) every libguestfs_query_mins minutes ~name:"new libguestfs version" ( fun () -> - git_force "https://github.com/libguestfs/libguestfs.git" "master"; - + git_force "master"; let version = git_latest_version "master" in require (website_updated version) ) diff --git a/utils.ml b/utils.ml index 25fc740..43119ee 100644 --- a/utils.ml +++ b/utils.ml @@ -16,3 +16,10 @@ let sort_uniq ?(cmp = Pervasives.compare) xs = let xs = List.sort cmp xs in let xs = uniq ~cmp xs in xs + +let rec filter_map f = function + | [] -> [] + | x :: xs -> + match f x with + | Some y -> y :: filter_map f xs + | None -> filter_map f xs