1 (* Memory info for virtual domains.
2 (C) Copyright 2008 Richard W.M. Jones, Red Hat Inc.
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 (* This is a script which downloads kernels from Fedora and extracts
21 the kernel layout information.
23 The basic plan is as follows:
25 (1) Use koji to list out all kernel builds, compare this to
26 what we have already got (in the kernels/ database at the top level
27 of the virt-mem source), and download any kernels we haven't
30 (2) For each kernel, get the kernel-*debuginfo* RPMs (there will
31 be several, one for each architecture, and one for each variant
34 (3) For each debuginfo RPM, extract the 'vmlinux' (kernel image)
35 from the RPM. This contains debugging symbols.
37 (4) Run 'pahole -E' (from acme's dwarves library) to extract all
38 the kernel structures.
40 (5) Save the kernel name/version/architecture + the output of pahole
41 in the kernels/ directory (the kernels database).
48 let (//) = Filename.concat
50 (* Wrappers around the XMLRPC calls. *)
52 package_name : string; (* eg. "kernel" *)
53 version : string; (* eg. "2.6.25" *)
54 release : string; (* eg. "1.fc8" *)
58 let string_of_build { package_name = package_name;
59 version = version; release = release;
60 build_id = build_id } =
61 sprintf "%d: %s %s %s" build_id package_name version release
64 rpm_id : int; (* RPM ID (for downloading, etc.) *)
66 rpm_name : string; (* eg. "kernel" *)
67 rpm_version : string; (* eg. "2.6.25" *)
68 rpm_release : string; (* eg. "1.fc8" *)
69 rpm_size : int; (* size in bytes of the RPM. *)
70 rpm_arch : string; (* architecture *)
73 let string_of_rpm { rpm_id = id; rpm_build = { build_id = build_id };
75 rpm_version = version; rpm_release = release;
76 rpm_size = size; rpm_arch = arch } =
77 sprintf "%d: (build %d) %s %s %s (%d bytes) %s"
78 id build_id name version release size arch
80 let get_string_from_struct name items =
81 match List.assoc name items with
83 | _ -> invalid_arg (name ^ ": expected string type")
85 let get_int_from_struct name items =
86 match List.assoc name items with
88 | _ -> invalid_arg (name ^ ": expected int type")
90 let koji_list_builds rpc ~prefix =
91 let builds = rpc#call "listBuilds" [
93 (* __starstar is some wierd Python thing which is needed for
94 * Python optional arguments to work.
97 "prefix", `String prefix;
107 let package_name = get_string_from_struct "package_name" items in
108 let version = get_string_from_struct "version" items in
109 let release = get_string_from_struct "release" items in
110 let build_id = get_int_from_struct "build_id" items in
111 { package_name = package_name;
112 version = version; release = release;
113 build_id = build_id }
116 prerr_endline "missing element in build structure from koji listBuilds() calls";
118 | Invalid_argument err ->
123 prerr_endline "unexpected type from koji listBuilds() call";
124 prerr_endline (XmlRpc.dump t);
128 prerr_endline "unexpected type from koji listBuilds() call:";
129 prerr_endline (XmlRpc.dump t);
132 let koji_list_build_rpms rpc ({ build_id = build_id } as build) =
133 let rpms = rpc#call "listBuildRPMs" [ `Int build_id ] in
141 let name = get_string_from_struct "name" items in
142 let version = get_string_from_struct "version" items in
143 let release = get_string_from_struct "release" items in
144 let build_id' = get_int_from_struct "build_id" items in
145 let id = get_int_from_struct "id" items in
146 let size = get_int_from_struct "size" items in
147 let arch = get_string_from_struct "arch" items in
148 assert (build_id = build_id');
149 { rpm_name = name; rpm_version = version; rpm_release = release;
150 rpm_build = build; rpm_id = id; rpm_size = size;
154 prerr_endline "missing element in build structure from koji listBuildRPMs() calls";
156 | Invalid_argument err ->
161 prerr_endline "unexpected type from koji listBuildRPMs() call";
162 prerr_endline (XmlRpc.dump t);
166 prerr_endline "unexpected type from koji listBuildRPMs() call:";
167 prerr_endline (XmlRpc.dump t);
170 (* This gets the RPM download URL for an RPM. I can't see a way to
171 * get this using the Koji API, but the URLs are fairly predictable
174 let koji_rpm_download_url { rpm_build = { package_name = build_name };
176 rpm_version = version; rpm_release = release;
178 let filename = sprintf "%s-%s-%s.%s.rpm" rpm_name version release arch in
179 let uri = sprintf "http://koji.fedoraproject.org/packages/%s/%s/%s/%s/%s"
180 build_name version release arch filename in
185 let rpc = new XmlRpc.client "http://koji.fedoraproject.org/kojihub" in
187 (* Grab the list of kernel builds from Koji. *)
188 printf "Downloading list of kernel builds from Koji ...\n%!";
189 let builds = koji_list_builds rpc ~prefix:"kernel" in
191 (* Only care about "kernel" and "kernel-xen" builds. *)
192 let builds = List.filter (
193 fun { package_name = name } ->
194 name = "kernel" || name = "kernel-xen"
197 let nr_builds = List.length builds in
198 printf "%d kernel builds found on Koji.\n%!" nr_builds;
200 (* Sort the builds by build ID in reverse, so that we tend to download
201 * the most recent kernels first.
204 let cmp { build_id = id1 } { build_id = id2 } = compare id2 id1 in
205 List.sort ~cmp builds in
209 printf "Build %d/%d: %s\n" (i+1) nr_builds (string_of_build build);
211 (* List the RPMs in the build. *)
212 let rpms = koji_list_build_rpms rpc build in
214 (* Only care about debuginfo builds, and not debuginfo-common. *)
215 let contains_string substr name =
216 try ignore (String.find name substr); true
217 with Invalid_string -> false
219 let contains_debuginfo = contains_string "debuginfo" in
220 let contains_common = contains_string "common" in
221 let rpms = List.filter (
222 fun { rpm_name = name } ->
223 contains_debuginfo name && not (contains_common name)
228 let uri, filename = koji_rpm_download_url rpm in
229 let infofile = outputdir // filename ^ ".info" in
232 try ignore (Unix.access infofile [Unix.F_OK]); true
233 with Unix.Unix_error _ -> false in
236 printf "Skipping %s\n%!" (string_of_rpm rpm)
238 printf "%s\n%!" (string_of_rpm rpm);
241 let r = Sys.command cmd in
243 failwith (sprintf "%s: command exited with code %d" cmd r)
246 (* Function to clean up the RPM & the temporary subdirectory
247 * (usr/, used for unpacking the RPM).
250 (try Unix.unlink filename with _ -> ());
251 ignore (Sys.command "rm -rf usr/")
257 Std.finally cleanup (
261 * Could use ocurl here (the OCaml CURL library) but
262 * using CURL as a library is generally more trouble
263 * than it's worth. So shell out to 'wget' instead.
265 printf "Downloading RPM ...\n%!";
266 run (sprintf "wget --quiet %s" (Filename.quote uri));
268 printf "Finished downloading RPM.\n%!";
270 (* Unpack vmlinux binary from the RPM. *)
271 run (sprintf "rpm2cpio %s | cpio -id --quiet '*/vmlinux'"
272 (Filename.quote filename));
274 run (sprintf "find usr/ -name vmlinux -print0 |
275 xargs -0 pahole -E > %s.data"
276 (Filename.quote outputdir // Filename.quote filename));
278 let chan = open_out infofile in
279 fprintf chan "Source: fedora-koji\n";
280 fprintf chan "Distribution: Fedora\n";
281 fprintf chan "RPM_id: %d\n" rpm.rpm_id;
282 fprintf chan "RPM_build_id: %d\n" rpm.rpm_build.build_id;
283 fprintf chan "Name: %s\n" rpm.rpm_name;
284 fprintf chan "Version: %s\n" rpm.rpm_version;
285 fprintf chan "Release: %s\n" rpm.rpm_release;
286 fprintf chan "Architecture: %s\n" rpm.rpm_arch;
287 fprintf chan "RPM_size: %d\n" rpm.rpm_size;
291 run (sprintf "rpm -qip %s >> %s"
292 (Filename.quote filename) (Filename.quote infofile));
296 eprintf "%s\n%!" msg (* but continue to next RPM ... *)
304 (* Create a temporary work directory, chdir into there to run the
305 * main program, then ensure that the temporary directory is cleaned
308 let olddir = Unix.getcwd () in
310 sprintf "%s/tmp%d%Ld"
311 Filename.temp_dir_name
312 (Unix.getpid ()) (Random.int64 Int64.max_int) in
314 Unix.mkdir tmpdir 0o700;
319 ignore (Sys.command (sprintf "rm -rf %s" (Filename.quote tmpdir)))
322 Std.finally cleanup (fun () -> main olddir) ()