X-Git-Url: http://git.annexia.org/?a=blobdiff_plain;f=libguestfs_upstream.ml;h=685c1b59de59f31dc996716ff487b49eba4f308c;hb=5324bda8dbb31613640901bfba9b0dd339bc9616;hp=eb6806efa954b30b0635a68dbd31ecec5e8357a5;hpb=1176fdbc7e636a9597e2294fef5574f2cd44f93e;p=goaljobs-goals.git

diff --git a/libguestfs_upstream.ml b/libguestfs_upstream.ml
index eb6806e..685c1b5 100644
--- a/libguestfs_upstream.ml
+++ b/libguestfs_upstream.ml
@@ -11,6 +11,7 @@
 open Goaljobs
 open Printf
 open Config
+open Libguestfs
 
 (* Enable debugging. *)
 let () =
@@ -20,84 +21,82 @@ let () =
 (* Log program output. *)
 let from = "rjones@redhat.com"
 let to_ = "rjones@redhat.com"
+(*
 let logfile = log_program_output ()
 let () = eprintf "logging to %s\n%!" logfile
+*)
 
 let package = "libguestfs"
 
-(* Helper object which stores everything about a version. *)
-type info = {
-  version : string;              (* The version as a normal string. *)
-  major : int;                   (* Broken-out version fields. *)
-  minor : int;
-  release: int;
-  is_stable : bool;              (* is a stable version of libguestfs? *)
-  branch : string;               (* 'master' or 'stable-1.xx' *)
-  package_version : string;      (* package-version *)
-  tarball : string;              (* package-version.tar.gz *)
-  urlpath : string;              (* download/1.X-(stable|development)/tarball *)
-  url : string;                  (* full download URL of tarball *)
-}
-
-(* Helper: Fetch latest gnulib into $buildtmp/repos/gnulib
- * XXX Move to Gnulib module.
- *)
-let get_gnulib () =
-  sh "
-    cd %s/repos
-    if [ ! -d gnulib ]; then git clone git://git.sv.gnu.org/gnulib.git; fi
-    cd gnulib
-    git checkout --force master
-    git pull
-  " buildtmp
-
 (* Goal: the website has been updated to 'version'. *)
 let rec goal website_updated version =
-  target (url_exists version.url);
+  target (url_exists version.url &&
+            (if version.is_stable then
+               url_exists version.python_url
+             else true)
+         );
 
   require (tarball_created version);
   require (tarball_tested version);
+  if version.is_stable then
+    require (tarball_signed version);
+
+  (* Python sdists only generated for stable releases. *)
+  if version.is_stable then (
+    require (python_tarball_created version);
+    require (python_tarball_tested version)
+  );
 
   (* We only update the website for the development releases. *)
   if not version.is_stable then
     require (website_built version);
 
-  require (website_cvs_checked_in version);
+  require (website_checked_in version);
+  if version.is_stable then
+    require (python_website_checked_in version);
   require (website_rsync_done version)
 
 (* Goal: website has been rsync'd. *)
 and website_rsync_done version =
-  let key = sprintf "libguestfs_website_rsync_done_%s" version.version in
-  target (memory_exists key);
-  onrun (fun () -> memory_set key "1");
+  target (url_contains_string "http://libguestfs.org" version.version &&
+          url_exists version.url);
 
   sh "
     cd %s
-    echo NOT RUNNING: . .rsync
-  " libguestfs_website_cvs
+    ./.rsync
+  " websites_repo
 
-(* Goal: Tarball added to CVS and CVS checked in. *)
-and website_cvs_checked_in version =
-  let key = sprintf "libguestfs_website_cvs_checked_in_%s" version.version in
+(* Goal: Tarball added to repository and checked in. *)
+and website_checked_in version =
+  let key = sprintf "libguestfs_website_checked_in_%s" version.version in
   target (memory_exists key);
   onrun (fun () -> memory_set key "1");
 
   require (tarball_created version);
   require (tarball_tested version);
+  if version.is_stable then
+    require (tarball_signed version);
 
   sh "
     cd %s
     cp %s/tarballs/%s %s
-    echo NOT RUNNING: cvs add -kb %s
-    echo NOT RUNNING: cvs ci -m \"Version %s\"
-  " libguestfs_website_cvs
+    if %b; then cp %s/tarballs/%s.sig %s.sig; fi
+    git add %s
+    cd %s
+    git add *.txt *.html
+    cd %s
+    git commit -m \"Version %s\"
+  " libguestfs_download_repo
     buildtmp version.tarball version.urlpath
+    version.is_stable buildtmp version.tarball version.urlpath
     version.urlpath
+    libguestfs_website_repo
+    websites_repo
     version.version
 
 (* Goal: website (local copy) has been built. *)
 and website_built version =
-  let index_file = sprintf "%s/index.html" libguestfs_website_cvs in
+  let index_file = sprintf "%s/index.html" libguestfs_website_repo in
   target (file_contains_string index_file version.version);
 
   require (tarball_created version);
@@ -115,12 +114,23 @@ and website_built version =
     echo %s > localenv
 
     ./localconfigure
-    make
-    make website
+    make V=1
+    make maintainer-upload-website WEBSITESDIR=%s
   " buildtmp version.tarball
     version.package_version
     (quote (libguestfs_localconfigure `Tarball))
-    (quote libguestfs_localenv)
+    (quote (libguestfs_localenv (supermin version)))
+    (quote websites_repo)
+
+(* Goal: for stable versions, the tarball must be signed. *)
+and tarball_signed version =
+  let tar_file = sprintf "%s/tarballs/%s" buildtmp version.tarball in
+  let sig_file = tar_file ^ ".sig" in
+  target (file_exists sig_file);
+
+  require (tarball_created version);
+
+  sh "sign-tarball %s" tar_file
 
 (* Goal: the tarball has passed the required set of tests before
  * a release is allowed.
@@ -141,20 +151,34 @@ and tarball_tested version =
     echo %s > localenv
 
     ./localconfigure
-    make
-    make check-release
+    make V=1
+    if ! make check-release; then
+      for f in `find -name test-suite.log | xargs grep -l ^FAIL:`; do
+        echo \"*** $f ***\"
+        cat $f
+      done
+      exit 1
+    fi
   " buildtmp version.tarball
     version.package_version
     (quote (libguestfs_localconfigure `Tarball))
-    (quote libguestfs_localenv)
+    (quote (libguestfs_localenv (supermin version)))
 
 (* Goal: the tarball has been created from git. *)
 and tarball_created version =
   let filename = sprintf "%s/tarballs/%s" buildtmp version.tarball in
   target (file_exists filename);
 
+  require (repo_up_to_date version.branch);
+
   let repodir = sprintf "%s/repos/%s-%s" buildtmp package version.branch in
-  require (directory_exists repodir);
+
+  (* Branches <= 1.32 are tagged with "1.32.11",
+   * branches >= 1.33 are tagged with "v1.33.11".
+   *)
+  let version_tag =
+    if version.minor >= 33 then "v" ^ version.version
+    else version.version in
 
   sh "
     cp -a %s libguestfs
@@ -166,13 +190,25 @@ and tarball_created version =
     echo %s > localenv
 
     ./localconfigure
-    make
+
+    # Ensure the po-docs are updated.  Grrr this is ugly ...
+    make ||:
+    rm po-docs/podfiles
+    make -C po-docs update-po
+
+    make V=1
     make dist
+
+    # Ensure redhat hardening flags didn't leak into the tarball.
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1214506
+    if zcat %s | grep -q redhat-hardened; then exit 1; fi
+
     mv %s %s/tarballs/%s
   " repodir
-    version.version
+    version_tag
     (quote (libguestfs_localconfigure `Git))
-    (quote libguestfs_localenv)
+    (quote (libguestfs_localenv (supermin version)))
+    version.tarball
     version.tarball buildtmp version.tarball
 
 (* Goal: test a commit. *)
@@ -180,15 +216,16 @@ and commit_tested branch commit =
   onfail (
     fun _ ->
       let subject = sprintf "goal: %s: FAILED" goalname in
-      mailto ~from ~subject ~attach:[logfile] to_
+      mailto ~from ~subject (*~attach:[logfile]*) to_
   );
 
   let key = sprintf "libguestfs_commit_tested_%s" commit in
   target (memory_exists key);
   onrun (fun () -> memory_set key "1");
 
+  require (repo_up_to_date branch);
+
   let repodir = sprintf "%s/repos/%s-%s" buildtmp package branch in
-  require (directory_exists repodir);
 
   sh "
     cp -a %s libguestfs
@@ -200,84 +237,83 @@ and commit_tested branch commit =
     echo %s > localenv
 
     ./localconfigure
-    make
-    make check-release
+    make V=1
+    if ! make check-release; then
+      for f in `find -name test-suite.log | xargs grep -l ^FAIL:`; do
+        echo \"*** $f ***\"
+        cat $f
+      done
+      exit 1
+    fi
   " repodir
     commit
     (quote (libguestfs_localconfigure `Git))
-    (quote libguestfs_localenv)
+    (quote (libguestfs_localenv None))
 
-(* Helper function to make a full 'info' object from a version
- * number.
- *)
-let vernames version =
-  Scanf.sscanf version "%d.%d.%d" (
-    fun major minor release ->
-      let is_stable = minor mod 2 = 0 in
-      let branch =
-        if is_stable then
-          sprintf "stable-%d.%d" major minor
-        else
-          sprintf "master" in
-      let package_version = sprintf "%s-%d.%d.%d" package major minor release in
-      let tarball = sprintf "%s.tar.gz" package_version in
-      let urlpath =
-        if is_stable then
-          sprintf "download/%d.%d-stable/%s" major minor tarball
-        else
-          sprintf "download/%d.%d-development/%s" major minor tarball in
-      let url = "http://libguestfs.org/" ^ urlpath in
-      { version = version;
-        major = major; minor = minor; release = release;
-        is_stable = is_stable;
-        branch = branch;
-        package_version = package_version;
-        tarball = tarball;
-        urlpath = urlpath;
-        url = url }
-  )
+and repo_up_to_date branch =
+  git_force branch
+
+(* Goals for Python sdists. *)
+and python_website_checked_in version =
+  let key = sprintf "libguestfs_python_website_checked_in_%s" version.version in
+  target (memory_exists key);
+  onrun (fun () -> memory_set key "1");
+
+  require (python_tarball_created version);
+  require (python_tarball_tested version);
 
-(* Helper function to read the latest version in a repo and return
- * the version.
- *)
-let git_latest_version branch =
-  let v = shout "
-    cd %s/repos/%s-%s
-    git describe --tags --abbrev=0
-  " buildtmp package (quote branch) in
-  vernames v
-
-(* Get the latest commit. *)
-let git_latest_commit branch =
-  shout "
-    cd %s/repos/%s-%s
-    git rev-parse HEAD
-  " buildtmp package (quote branch)
-
-(* Clone or update a repo to the latest version on a branch, by force.
- * It is cached in name = $buildtmp/repos/<package>-<branch>
- *)
-let git_force url branch =
   sh "
-    cd %s/repos
-    if [ ! -d %s-%s ]; then git clone %s %s-%s; fi
-    cd %s-%s
-    git checkout --force %s
-    git pull
-    # Copy or update gnulib
-    git submodule init
-    git submodule update
-  " buildtmp
-    package (quote branch) (quote url) package (quote branch)
-    package (quote branch)
-    (quote branch)
+    cd %s
+    cp %s/tarballs/%s %s
+    git add %s
+    git commit -m \"Python sdist version %s\"
+  " libguestfs_download_repo
+    buildtmp version.python_tarball version.python_urlpath
+    version.python_urlpath
+    version.version
+
+and python_tarball_tested version =
+  let key = sprintf "libguestfs_python_tarball_tested_%s" version.version in
+  target (memory_exists key);
+  onrun (fun () -> memory_set key "1");
+
+  require (python_tarball_created version);
+
+  sh "
+    virtualenv venv
+    source ./venv/bin/activate
+    pip install %s/tarballs/%s
+  " buildtmp version.python_tarball
+
+and python_tarball_created version =
+  let filename = sprintf "%s/tarballs/%s" buildtmp version.python_tarball in
+  target (file_exists filename);
+
+  require (tarball_created version);
+
+  sh "
+    tar zxf %s/tarballs/%s
+    cd %s
+
+    echo %s > localconfigure
+    chmod +x localconfigure
+    echo %s > localenv
+
+    ./localconfigure
+    make V=1
+    make -C python sdist
+    cp python/dist/%s %s
+  " buildtmp version.tarball
+    version.package_version
+    (quote (libguestfs_localconfigure `Tarball))
+    (quote (libguestfs_localenv (supermin version)))
+    version.python_tarball filename
 
 let () =
   (* Add a periodic job to check for new git commits and test them. *)
   every libguestfs_query_mins minutes ~name:"new libguestfs commit" (
     fun () ->
-      git_force "https://github.com/libguestfs/libguestfs.git" "master";
-
+      require (repo_up_to_date "master");
       let commit = git_latest_commit "master" in
       require (commit_tested "master" commit);
   );
@@ -285,8 +321,7 @@ let () =
   (* Periodic job to build new tarballs. *)
   every libguestfs_query_mins minutes ~name:"new libguestfs version" (
     fun () ->
-      git_force "https://github.com/libguestfs/libguestfs.git" "master";
-
+      require (repo_up_to_date "master");
       let version = git_latest_version "master" in
       require (website_updated version)
   )