From 0f89ba0654de234429042ffcc91c8a0de94ec98b Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 20 Aug 2010 21:31:49 +0100 Subject: [PATCH] Add -f checksum mode to allow caching of appliances. --- helper/Makefile.am | 1 + helper/appliance.c | 5 +- helper/checksum.c | 117 +++++++++++++++++++++++++++++++++ helper/cpio.c | 2 +- helper/ext2.c | 2 +- helper/febootstrap-supermin-helper.pod | 24 +++++-- helper/helper.h | 7 +- helper/kernel.c | 16 +++-- helper/main.c | 34 ++++++---- 9 files changed, 178 insertions(+), 30 deletions(-) create mode 100644 helper/checksum.c diff --git a/helper/Makefile.am b/helper/Makefile.am index 96b5581..940a2ee 100644 --- a/helper/Makefile.am +++ b/helper/Makefile.am @@ -23,6 +23,7 @@ bin_PROGRAMS = \ febootstrap_supermin_helper_SOURCES = \ helper.h \ appliance.c \ + checksum.c \ cpio.c \ ext2.c \ ext2cpio.c \ diff --git a/helper/appliance.c b/helper/appliance.c index 2474b14..4cbebf4 100644 --- a/helper/appliance.c +++ b/helper/appliance.c @@ -65,14 +65,15 @@ static void add_hostfiles (const char *hostfiles_file, struct writer *); * hostfiles) or use a directory to store these files. */ void -create_appliance (char **inputs, int nr_inputs, +create_appliance (const char *hostcpu, + char **inputs, int nr_inputs, const char *whitelist, const char *modpath, const char *initrd, const char *appliance, struct writer *writer) { - writer->wr_start (appliance, modpath, initrd); + writer->wr_start (hostcpu, appliance, modpath, initrd); iterate_inputs (inputs, nr_inputs, writer); diff --git a/helper/checksum.c b/helper/checksum.c new file mode 100644 index 0000000..337134c --- /dev/null +++ b/helper/checksum.c @@ -0,0 +1,117 @@ +/* febootstrap-supermin-helper reimplementation in C. + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "error.h" + +#include "helper.h" + +static FILE *pp = NULL; + +/* This is the command we run to calculate the SHA. Note that we sort + * the rows first so that the checksum is roughly stable, since the + * order that we output files might not be (eg. because we rely on the + * ordering of readdir). Uncomment the second line to see the output + * before hashing. + */ +static const char *shacmd = "sort | sha256sum | awk '{print $1}'"; +//static const char *shacmd = "sort | cat"; + +static void +checksum_start (const char *hostcpu, const char *appliance, + const char *modpath, const char *initrd) +{ + pp = popen (shacmd, "w"); + if (pp == NULL) + error (EXIT_FAILURE, errno, "popen: command failed: %s", shacmd); + + fprintf (pp, "%s %s %s %d\n", + PACKAGE_STRING, hostcpu, modpath, geteuid ()); +} + +static void +checksum_end (void) +{ + if (pclose (pp) == -1) + error (EXIT_FAILURE, errno, "pclose: command failed: %s", shacmd); + pp = NULL; +} + +static void +checksum_file_stat (const char *filename, const struct stat *statbuf) +{ + /* Publically writable directories (ie. /tmp) don't have stable + * times. Since we only care about some attributes of directories + * in any case, we vary the output accordingly. + */ + if (!S_ISDIR (statbuf->st_mode)) + fprintf (pp, "%s %ld %ld %d %d %ld %o\n", + filename, + (long) statbuf->st_ctime, (long) statbuf->st_mtime, + statbuf->st_uid, statbuf->st_gid, statbuf->st_size, + statbuf->st_mode); + else + fprintf (pp, "%s %d %d %o\n", + filename, + statbuf->st_uid, statbuf->st_gid, + statbuf->st_mode); +} + +static void +checksum_file (const char *filename) +{ + struct stat statbuf; + + if (lstat (filename, &statbuf) == -1) + error (EXIT_FAILURE, errno, "lstat: %s", filename); + checksum_file_stat (filename, &statbuf); +} + +static void +checksum_fts_entry (FTSENT *entry) +{ + if (entry->fts_info & FTS_NS || entry->fts_info & FTS_NSOK) + checksum_file (entry->fts_path); + else + checksum_file_stat (entry->fts_path, entry->fts_statp); +} + +static void +checksum_cpio_file (const char *cpio_file) +{ + checksum_file (cpio_file); +} + +struct writer checksum_writer = { + .wr_start = checksum_start, + .wr_end = checksum_end, + .wr_file = checksum_file, + .wr_file_stat = checksum_file_stat, + .wr_fts_entry = checksum_fts_entry, + .wr_cpio_file = checksum_cpio_file, +}; diff --git a/helper/cpio.c b/helper/cpio.c index 1dd22b0..0ca59a6 100644 --- a/helper/cpio.c +++ b/helper/cpio.c @@ -250,7 +250,7 @@ write_padding (size_t len) } static void -cpio_start (const char *appliance, +cpio_start (const char *hostcpu, const char *appliance, const char *modpath, const char *initrd) { out_fd = open (appliance, O_WRONLY | O_CREAT | O_TRUNC | O_NOCTTY, 0644); diff --git a/helper/ext2.c b/helper/ext2.c index 9d60da9..a27fb47 100644 --- a/helper/ext2.c +++ b/helper/ext2.c @@ -52,7 +52,7 @@ ext2_filsys fs; #define APPLIANCE_SIZE (1024*1024*1024) static void -ext2_start (const char *appliance, +ext2_start (const char *hostcpu, const char *appliance, const char *modpath, const char *initrd) { initialize_ext2_error_table (); diff --git a/helper/febootstrap-supermin-helper.pod b/helper/febootstrap-supermin-helper.pod index 7a3dbd1..c6c551c 100644 --- a/helper/febootstrap-supermin-helper.pod +++ b/helper/febootstrap-supermin-helper.pod @@ -9,6 +9,8 @@ febootstrap-supermin-helper - Reconstruct initramfs from supermin appliance. febootstrap-supermin-helper -f ext2 input [...] host_cpu kernel initrd appliance + febootstrap-supermin-helper -f checksum input [...] host_cpu + =head1 DESCRIPTION I reconstructs a bootable kernel and @@ -31,7 +33,8 @@ C, C and C are the temporary output files that this script produces. These output files are meant to be used just for booting the appliance, and should be deleted straight afterwards. The extra C parameter is only required when -the format is C. +the format is C. None of these parameters are needed for +the checksum output C<-f checksum>. =head1 OPTIONS @@ -54,9 +57,22 @@ and C, where the C is the appliance. An ext2 filesystem. -In this case you have to supply names for the C, -a small C which is used just to locate the appliance, -and the C (the ext2 filesystem). +In this case you have to supply names for the C, a small +C which is used just to locate the appliance, and the +C (the ext2 filesystem). + +=item checksum + +Output a checksum. + +This prints a checksum which only changes when one of the input files +changes. + +You can use this in order to cache the output of a previous run of +this program: computing the checksum is much quicker than building an +appliance, and you only need to invalidate the cache (and consequently +rebuild the appliance) when the checksum changes. Note that the +host_cpu and the UID of the current user are included in the checksum. =back diff --git a/helper/helper.h b/helper/helper.h index cb28695..e0d1fbb 100644 --- a/helper/helper.h +++ b/helper/helper.h @@ -28,7 +28,7 @@ struct writer { * 'initrd' is the mini-initrd to create (only used for ext2 output). * 'modpath' is the kernel module path. */ - void (*wr_start) (const char *appliance, + void (*wr_start) (const char *hostcpu, const char *appliance, const char *modpath, const char *initrd); /* Finish off the appliance. */ @@ -52,7 +52,10 @@ extern struct timeval start_t; extern int verbose; /* appliance.c */ -extern void create_appliance (char **inputs, int nr_inputs, const char *whitelist, const char *modpath, const char *initrd, const char *appliance, struct writer *writer); +extern void create_appliance (const char *hostcpu, char **inputs, int nr_inputs, const char *whitelist, const char *modpath, const char *initrd, const char *appliance, struct writer *writer); + +/* checksum.c */ +extern struct writer checksum_writer; /* cpio.c */ extern struct writer cpio_writer; diff --git a/helper/kernel.c b/helper/kernel.c index 3301826..0903805 100644 --- a/helper/kernel.c +++ b/helper/kernel.c @@ -122,16 +122,18 @@ create_kernel (const char *hostcpu, const char *kernel) sort (candidates, reverse_filevercmp); - /* Choose the first candidate. */ - char *tmp = xasprintf (KERNELDIR "/%s", candidates[0]); + if (kernel) { + /* Choose the first candidate. */ + char *tmp = xasprintf (KERNELDIR "/%s", candidates[0]); - if (verbose) - fprintf (stderr, "creating symlink %s -> %s\n", kernel, tmp); + if (verbose >= 2) + fprintf (stderr, "creating symlink %s -> %s\n", kernel, tmp); - if (symlink (tmp, kernel) == -1) - error (EXIT_FAILURE, errno, "symlink kernel"); + if (symlink (tmp, kernel) == -1) + error (EXIT_FAILURE, errno, "symlink kernel"); - free (tmp); + free (tmp); + } return get_modpath (candidates[0]); diff --git a/helper/main.c b/helper/main.c index b4359b6..4afcb24 100644 --- a/helper/main.c +++ b/helper/main.c @@ -58,6 +58,7 @@ usage (const char *progname) "Usage:\n" " %s [-options] inputs [...] host_cpu kernel initrd\n" " %s -f ext2 inputs [...] host_cpu kernel initrd appliance\n" + " %s -f checksum inputs [...] host_cpu\n" " %s --help\n" " %s --version\n" "\n" @@ -72,7 +73,7 @@ usage (const char *progname) "Options:\n" " --help\n" " Display this help text and exit.\n" - " -f cpio|ext2 | --format cpio|ext2\n" + " -f cpio|ext2|checksum | --format cpio|ext2|checksum\n" " Specify output format (default: cpio).\n" " -k file | --kmods file\n" " Specify kernel module whitelist.\n" @@ -80,7 +81,7 @@ usage (const char *progname) " Enable verbose messages (give multiple times for more verbosity).\n" " --version | -V\n" " Display version number and exit.\n", - progname, progname, progname, progname, progname); + progname, progname, progname, progname, progname, progname); } int @@ -135,8 +136,13 @@ main (int argc, char *argv[]) writer = &ext2_writer; nr_outputs = 3; /* kernel, initrd, appliance */ } + else if (strcmp (format, "checksum") == 0) { + writer = &checksum_writer; + nr_outputs = 0; /* (none) */ + } else { - fprintf (stderr, "%s: incorrect output format (-f): must be cpio|ext2\n", + fprintf (stderr, + "%s: incorrect output format (-f): must be cpio|ext2|checksum\n", argv[0]); exit (EXIT_FAILURE); } @@ -160,10 +166,11 @@ main (int argc, char *argv[]) const char *hostcpu = outputs[-1]; /* Output files. */ - const char *kernel = outputs[0]; - const char *initrd; - const char *appliance; - initrd = appliance = outputs[1]; + const char *kernel = NULL, *initrd = NULL, *appliance = NULL; + if (nr_outputs > 0) + kernel = outputs[0]; + if (nr_outputs > 1) + initrd = appliance = outputs[1]; if (nr_outputs > 2) appliance = outputs[2]; @@ -181,20 +188,21 @@ main (int argc, char *argv[]) } /* Remove the output files if they exist. */ - unlink (kernel); - unlink (initrd); - if (initrd != appliance) + if (kernel) + unlink (kernel); + if (initrd) + unlink (initrd); + if (appliance && initrd != appliance) unlink (appliance); /* Create kernel output file. */ - const char *modpath; - modpath = create_kernel (hostcpu, kernel); + const char *modpath = create_kernel (hostcpu, kernel); if (verbose) print_timestamped_message ("finished creating kernel"); /* Create the appliance. */ - create_appliance (inputs, nr_inputs, whitelist, modpath, + create_appliance (hostcpu, inputs, nr_inputs, whitelist, modpath, initrd, appliance, writer); if (verbose) -- 1.8.3.1