From 9392e738c4c65284145e50644d463fc0aa05bbad Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Thu, 1 Jan 1970 00:00:00 +0000 Subject: [PATCH] Try to improve robustness, make kerneldb additions atomic. Move Koji code to separate module. --- extract/fedora-koji/.depend | 2 + extract/fedora-koji/Makefile.in | 2 +- .../fedora-koji/fedora_koji_download_kernels.ml | 44 ++++-- extract/fedora-koji/koji.ml | 154 +++++++++++++++++++++ extract/fedora-koji/koji.mli | 54 ++++++++ 5 files changed, 240 insertions(+), 16 deletions(-) create mode 100644 extract/fedora-koji/koji.ml create mode 100644 extract/fedora-koji/koji.mli diff --git a/extract/fedora-koji/.depend b/extract/fedora-koji/.depend index e69de29..2afb721 100644 --- a/extract/fedora-koji/.depend +++ b/extract/fedora-koji/.depend @@ -0,0 +1,2 @@ +koji.cmo: koji.cmi +koji.cmx: koji.cmi diff --git a/extract/fedora-koji/Makefile.in b/extract/fedora-koji/Makefile.in index 3154a4f..bf6edb3 100644 --- a/extract/fedora-koji/Makefile.in +++ b/extract/fedora-koji/Makefile.in @@ -40,7 +40,7 @@ else TARGETS = endif -OBJS = fedora_koji_download_kernels.cmo +OBJS = koji.cmo fedora_koji_download_kernels.cmo XOBJS = $(OBJS:.cmo=.cmx) all: byte opt diff --git a/extract/fedora-koji/fedora_koji_download_kernels.ml b/extract/fedora-koji/fedora_koji_download_kernels.ml index 57abfd3..df12f66 100644 --- a/extract/fedora-koji/fedora_koji_download_kernels.ml +++ b/extract/fedora-koji/fedora_koji_download_kernels.ml @@ -223,32 +223,37 @@ let main outputdir = contains_debuginfo name && not (contains_common name) ) rpms in - List.iter ( - fun rpm -> + let nr_rpms = List.length rpms in + + List.iteri ( + fun j rpm -> let uri, filename = koji_rpm_download_url rpm in - let infofile = outputdir // filename ^ ".info" in + let infoname = filename ^ ".info" in + let infopath = outputdir // infoname in let infoexists = - try ignore (Unix.access infofile [Unix.F_OK]); true + try ignore (Unix.access infopath [Unix.F_OK]); true with Unix.Unix_error _ -> false in if infoexists then printf "Skipping %s\n%!" (string_of_rpm rpm) else ( - printf "%s\n%!" (string_of_rpm rpm); + printf "%d/%d %d/%d %s\n%!" + (i+1) nr_builds (j+1) nr_rpms (string_of_rpm rpm); let run cmd = let r = Sys.command cmd in if r <> 0 then failwith (sprintf "%s: command exited with code %d" cmd r) in + let run cmd = ksprintf run cmd in (* Function to clean up the RPM & the temporary subdirectory * (usr/, used for unpacking the RPM). *) let cleanup () = (try Unix.unlink filename with _ -> ()); - ignore (Sys.command "rm -rf usr/") + ignore (Sys.command "rm -rf *.info *.data usr") in cleanup (); @@ -263,19 +268,19 @@ let main outputdir = * than it's worth. So shell out to 'wget' instead. *) printf "Downloading RPM ...\n%!"; - run (sprintf "wget --quiet %s" (Filename.quote uri)); + run "wget --quiet %s" (Filename.quote uri); printf "Finished downloading RPM.\n%!"; (* Unpack vmlinux binary from the RPM. *) - run (sprintf "rpm2cpio %s | cpio -id --quiet '*/vmlinux'" - (Filename.quote filename)); + run "rpm2cpio %s | cpio -id --quiet '*/vmlinux'" + (Filename.quote filename); - run (sprintf "find usr/ -name vmlinux -print0 | - xargs -0 pahole -E > %s.data" - (Filename.quote outputdir // Filename.quote filename)); + run "find usr/ -name vmlinux -print0 | + xargs -0 pahole -E > %s.data" + (Filename.quote filename); - let chan = open_out infofile in + let chan = open_out infoname in fprintf chan "Source: fedora-koji\n"; fprintf chan "Distribution: Fedora\n"; fprintf chan "RPM_id: %d\n" rpm.rpm_id; @@ -288,8 +293,17 @@ let main outputdir = fprintf chan "\n"; close_out chan; - run (sprintf "rpm -qip %s >> %s" - (Filename.quote filename) (Filename.quote infofile)); + run "rpm -qip %s >> %s" + (Filename.quote filename) (Filename.quote infoname); + + (* Atomically move the info & data files to their final + * destination. + *) + run "mv %s.data %s.data" + (Filename.quote filename) + (Filename.quote (outputdir // filename)); + run "mv %s %s" + (Filename.quote infoname) (Filename.quote infopath); ) () with Failure msg -> diff --git a/extract/fedora-koji/koji.ml b/extract/fedora-koji/koji.ml new file mode 100644 index 0000000..aa24483 --- /dev/null +++ b/extract/fedora-koji/koji.ml @@ -0,0 +1,154 @@ +(* Memory info for virtual domains. + (C) Copyright 2008 Richard W.M. Jones, Red Hat Inc. + http://libvirt.org/ + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*) + +open Printf + +type build = { + package_name : string; (* eg. "kernel" *) + version : string; (* eg. "2.6.25" *) + release : string; (* eg. "1.fc8" *) + build_id : int; +} + +let string_of_build { package_name = package_name; + version = version; release = release; + build_id = build_id } = + sprintf "%d: %s %s %s" build_id package_name version release + +type rpm = { + rpm_id : int; (* RPM ID (for downloading, etc.) *) + rpm_build : build; + rpm_name : string; (* eg. "kernel" *) + rpm_version : string; (* eg. "2.6.25" *) + rpm_release : string; (* eg. "1.fc8" *) + rpm_size : int; (* size in bytes of the RPM. *) + rpm_arch : string; (* architecture *) +} + +let string_of_rpm { rpm_id = id; rpm_build = { build_id = build_id }; + rpm_name = name; + rpm_version = version; rpm_release = release; + rpm_size = size; rpm_arch = arch } = + sprintf "%d: (build %d) %s %s %s (%d bytes) %s" + id build_id name version release size arch + +type rpc = < call : string -> XmlRpc.value list -> XmlRpc.value > + +let get_string_from_struct name items = + match List.assoc name items with + | `String str -> str + | _ -> invalid_arg (name ^ ": expected string type") + +let get_int_from_struct name items = + match List.assoc name items with + | `Int i -> i + | _ -> invalid_arg (name ^ ": expected int type") + +let list_builds rpc ~prefix = + let builds = rpc#call "listBuilds" [ + `Struct [ + (* __starstar is some wierd Python thing which is needed for + * Python optional arguments to work. + *) + "__starstar", `Int 1; + "prefix", `String prefix; + ] + ] in + + match builds with + | `Array builds -> + List.map ( + function + | `Struct items -> + (try + let package_name = get_string_from_struct "package_name" items in + let version = get_string_from_struct "version" items in + let release = get_string_from_struct "release" items in + let build_id = get_int_from_struct "build_id" items in + { package_name = package_name; + version = version; release = release; + build_id = build_id } + with + | Not_found -> + prerr_endline "missing element in build structure from koji listBuilds() calls"; + exit 1 + | Invalid_argument err -> + prerr_endline err; + exit 1 + ) + | t -> + prerr_endline "unexpected type from koji listBuilds() call"; + prerr_endline (XmlRpc.dump t); + exit 1 + ) builds + | t -> + prerr_endline "unexpected type from koji listBuilds() call:"; + prerr_endline (XmlRpc.dump t); + exit 1 + +let list_build_rpms rpc ({ build_id = build_id } as build) = + let rpms = rpc#call "listBuildRPMs" [ `Int build_id ] in + + match rpms with + | `Array rpms -> + List.map ( + function + | `Struct items -> + (try + let name = get_string_from_struct "name" items in + let version = get_string_from_struct "version" items in + let release = get_string_from_struct "release" items in + let build_id' = get_int_from_struct "build_id" items in + let id = get_int_from_struct "id" items in + let size = get_int_from_struct "size" items in + let arch = get_string_from_struct "arch" items in + assert (build_id = build_id'); + { rpm_name = name; rpm_version = version; rpm_release = release; + rpm_build = build; rpm_id = id; rpm_size = size; + rpm_arch = arch } + with + | Not_found -> + prerr_endline "missing element in build structure from koji listBuildRPMs() calls"; + exit 1 + | Invalid_argument err -> + prerr_endline err; + exit 1 + ) + | t -> + prerr_endline "unexpected type from koji listBuildRPMs() call"; + prerr_endline (XmlRpc.dump t); + exit 1 + ) rpms + | t -> + prerr_endline "unexpected type from koji listBuildRPMs() call:"; + prerr_endline (XmlRpc.dump t); + exit 1 + +(* This gets the RPM download URL for an RPM. I can't see a way to + * get this using the Koji API, but the URLs are fairly predictable + * anyway. + *) +let rpm_download_url { rpm_build = { package_name = build_name }; + rpm_name = rpm_name; + rpm_version = version; rpm_release = release; + rpm_arch = arch } = + let filename = sprintf "%s-%s-%s.%s.rpm" rpm_name version release arch in + let uri = sprintf "http://koji.fedoraproject.org/packages/%s/%s/%s/%s/%s" + build_name version release arch filename in + uri, filename diff --git a/extract/fedora-koji/koji.mli b/extract/fedora-koji/koji.mli new file mode 100644 index 0000000..a1e1188 --- /dev/null +++ b/extract/fedora-koji/koji.mli @@ -0,0 +1,54 @@ +(* Memory info for virtual domains. + (C) Copyright 2008 Richard W.M. Jones, Red Hat Inc. + http://libvirt.org/ + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*) + +(** Wrappers around the XMLRPC calls. *) + +type build = { + package_name : string; (* eg. "kernel" *) + version : string; (* eg. "2.6.25" *) + release : string; (* eg. "1.fc8" *) + build_id : int; +} + +type rpm = { + rpm_id : int; (* RPM ID (for downloading, etc.) *) + rpm_build : build; + rpm_name : string; (* eg. "kernel" *) + rpm_version : string; (* eg. "2.6.25" *) + rpm_release : string; (* eg. "1.fc8" *) + rpm_size : int; (* size in bytes of the RPM. *) + rpm_arch : string; (* architecture *) +} + +val string_of_build : build -> string + +val string_of_rpm : rpm -> string + +(* Zzzaaargghh why doesn't xmlrpc-light define a name for the client object? *) +type rpc = < call : string -> XmlRpc.value list -> XmlRpc.value > + +val list_builds : rpc -> prefix:string -> build list + +val list_build_rpms : rpc -> build -> rpm list + +(* This gets the RPM download URL for an RPM. I can't see a way to + * get this using the Koji API, but the URLs are fairly predictable + * anyway. The return value is (URI, filename). + *) +val rpm_download_url : rpm -> string * string -- 1.8.3.1