From f71a10c6282c56aac97e8b1651e018e5d63c7a4b Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 10 Aug 2009 14:34:24 +0100 Subject: [PATCH] Hostinfo day 5: Implement first round of static host commands. --- README | 2 +- conf/guests.conf.in | 13 +++- conf/hostinfo.conf.in | 4 + configure.ac | 2 +- hostinfod/Makefile.am | 4 +- hostinfod/commands.c | 42 +++++++++- hostinfod/configuration.c | 127 +++++++++++++++++++++++++++--- hostinfod/hostinfo-protocol.pod | 166 +++++++++++++++++++++++++++++++++++++++ hostinfod/hostinfod.h | 15 +++- hostinfod/main.c | 16 +++- hostinfod/nodeinfo.c | 168 ++++++++++++++++++++++++++++++++++++++++ hostinfod/ping.c | 3 +- hostinfod/virt.c | 54 +++++++++++++ 13 files changed, 595 insertions(+), 21 deletions(-) create mode 100644 hostinfod/nodeinfo.c create mode 100644 hostinfod/virt.c diff --git a/README b/README index 6a6f62b..2deb984 100644 --- a/README +++ b/README @@ -21,7 +21,7 @@ Requirements - pod2man and other perldoc tools * these are usually supplied with Perl -- libvirt +- libvirt & development tools - Perl module Sys::Virt diff --git a/conf/guests.conf.in b/conf/guests.conf.in index 0e2cf31..a822be2 100644 --- a/conf/guests.conf.in +++ b/conf/guests.conf.in @@ -20,6 +20,17 @@ #interval 60 # Catch-all default rule for guests. This rule MUST be last in the file. +# This lists commands that are enabled. Any commands not listed here +# will be disabled by default. [*] interval 1 -physcpus on +availcpus on +corespersocket on +memory on +mhz on +model on +nodes on +physcpus on +ping on +socketspernode on +threadspercore on diff --git a/conf/hostinfo.conf.in b/conf/hostinfo.conf.in index aad6197..fa33d72 100644 --- a/conf/hostinfo.conf.in +++ b/conf/hostinfo.conf.in @@ -17,6 +17,10 @@ guests @sysconfdir@/hostinfo/guests.conf # change this. #sockets @localstatedir@/lib/hostinfo +# Libvirt connection URI. +# See: http://libvirt.org/uri.html +#libvirt xen:/// + # Enable verbose messages sent to the system log files. #verbose 1 diff --git a/configure.ac b/configure.ac index cc05e37..7a1377e 100644 --- a/configure.ac +++ b/configure.ac @@ -37,7 +37,7 @@ dnl C functions which may be missing on older systems. AC_CHECK_FUNCS([inotify_init1 clock_gettime]) dnl Check for required packages using pkg-config. -PKG_CHECK_MODULES([HOSTINFOD],[apr-1 >= 1.3]) +PKG_CHECK_MODULES([HOSTINFOD],[apr-1 >= 1.3 libvirt >= 0.2.1]) dnl Check for Perl and POD. AC_CHECK_PROG([PERL],[perl],[perl],[no]) diff --git a/hostinfod/Makefile.am b/hostinfod/Makefile.am index 5fc2cb1..63709db 100644 --- a/hostinfod/Makefile.am +++ b/hostinfod/Makefile.am @@ -28,7 +28,9 @@ hostinfod_SOURCES = \ hostinfod.h \ main.c \ monitor_sockets.c \ - ping.c + nodeinfo.c \ + ping.c \ + virt.c hostinfod_CFLAGS = \ -Wall \ diff --git a/hostinfod/commands.c b/hostinfod/commands.c index b928c36..cb66179 100644 --- a/hostinfod/commands.c +++ b/hostinfod/commands.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -61,6 +62,9 @@ execute_command (const struct timespec *now, apr_array_header_t *args; struct arg arg; command_fn fn; + int enabled; + double interval; + struct timespec *last; debug ("%s: %s", hval->name, command); @@ -208,8 +212,42 @@ execute_command (const struct timespec *now, /* Before dispatching the command, check the command is enabled * and guest is not calling it too frequently. */ - error ("XXXXXXX enabled check not implemented XXXXXXX"); - error ("XXXXXXX frequency check not implemented XXXXXXX"); + check_guests_file (hval, cmd, &interval, &enabled); + + if (!enabled) { + warning ("%s: guest tried disabled command '%s'", hval->name, cmd); + send_error (hval, 401); + return; + } + + last = apr_hash_get (hval->lasttime, cmd, APR_HASH_KEY_STRING); + if (last) { + struct timespec timediff; + double interval_int, interval_frac; + struct timespec interval_ts; + + diff_timespec (&timediff, now, last); + + interval_frac = modf (interval, &interval_int); + interval_ts.tv_sec = interval_int; + interval_ts.tv_nsec = interval_frac * 1000000000; + + debug ("%s: %s: interval %ds %ldns, time since last %ds %ldns", + hval->name, cmd, + (int) interval_ts.tv_sec, interval_ts.tv_nsec, + (int) timediff.tv_sec, timediff.tv_nsec); + + if (interval_ts.tv_sec > timediff.tv_sec || + (interval_ts.tv_sec == timediff.tv_sec && + interval_ts.tv_nsec > timediff.tv_nsec)) { + warning ("%s: command '%s' exceeded interval allowed", hval->name, cmd); + send_error (hval, 406); + return; + } + } + + last = apr_pmemdup (hval->pool, now, sizeof *now); + apr_hash_set (hval->lasttime, cmd, APR_HASH_KEY_STRING, last); /* Dispatch the command. */ fn (hval, cmd, args); diff --git a/hostinfod/configuration.c b/hostinfod/configuration.c index 90200f8..86a43dc 100644 --- a/hostinfod/configuration.c +++ b/hostinfod/configuration.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -31,26 +32,30 @@ #include "hostinfod.h" typedef int (*process_line_fn) (const char *path, int lineno, - const char *key, const char *value); + const char *key, const char *value, + void *data); typedef int (*process_section_fn) (const char *path, int lineno, - const char *section); + const char *section, + void *data); static void process_conf_file (const char *path, int exit_if_not_exist, process_line_fn process_line, - process_section_fn process_section); + process_section_fn process_section, + void *data); static int get_bool (const char *str); /* Read the main configuration file. */ -static int main_process_line (const char *path, int lineno, const char *key, const char *value); +static int main_process_line (const char *path, int lineno, const char *key, const char *value, void *data); void read_main_conf_file (void) { - process_conf_file (conf_file, 0, main_process_line, NULL); + process_conf_file (conf_file, 0, main_process_line, NULL, NULL); } static int main_process_line (const char *path, int lineno, - const char *key, const char *value) + const char *key, const char *value, + void *data) { int bool; @@ -66,6 +71,13 @@ main_process_line (const char *path, int lineno, return -1; } socket_dir = apr_pstrdup (pool, value); + } else if (strcasecmp (key, "libvirt") == 0) { + if (!value) { + error ("%s:%d: directive is empty: %s", path, lineno, key); + return -1; + } + if (!libvirt_uri_set_on_cmdline) + libvirt_uri = apr_pstrdup (pool, value); } else if (strcasecmp (key, "verbose") == 0) { bool = get_bool (value); if (bool == -1) { @@ -90,11 +102,102 @@ main_process_line (const char *path, int lineno, return 0; } +/* Check that 'cmd' is enabled for the named guest, and that + * it is not being called too frequently. + * + * XXX Rereading the configuration file each time is possibly + * inefficient. + * + * Returns 0 = proceed, -1 = fail. + */ +static int guests_process_line (const char *path, int lineno, + const char *key, const char *value, + void *data); +static int guests_process_section (const char *path, int lineno, + const char *section, + void *data); + +struct guests_data { + const char *name; /* guest "driver-name" */ + const char *cmd; /* command being tested */ + int in_section; /* currently processing the right section? */ + double interval; /* interval for this guest (0 = any) */ + int enabled; /* is command enabled? */ +}; + +void +check_guests_file (struct guest_description *hval, const char *cmd, + double *interval, int *enabled) +{ + struct guests_data *data = apr_palloc (hval->rpool, sizeof *data); + + data->name = hval->name; + data->cmd = cmd; + data->in_section = 0; + data->interval = 60.; /* default */ + data->enabled = 0; /* default */ + + process_conf_file (guests_file, 1, + guests_process_line, guests_process_section, data); + + *interval = data->interval; + *enabled = data->enabled; +} + +static int +guests_process_line (const char *path, int lineno, + const char *key, const char *value, + void *datav) +{ + struct guests_data *data = datav; + int bool; + + if (!data->in_section) + return 0; + + if (strcasecmp (key, "interval") == 0) { + if (strcasecmp (value, "any") == 0) + data->interval = 0; + else { + if (sscanf (value, "%lg", &data->interval) != 1) { + error ("%s:%d: %s: not a valid decimal number", path, lineno, key); + return -1; + } + } + } else if (strcasecmp (key, data->cmd) == 0) { + bool = get_bool (value); + if (bool == -1) { + error ("%s:%d: %s: not a valid boolean - use 1 or 0", path, lineno, key); + return -1; + } + data->enabled = bool; + } + + return 0; +} + +static int +guests_process_section (const char *path, int lineno, + const char *section, + void *datav) +{ + struct guests_data *data = datav; + int flags = 0; + +#ifdef FNM_CASEFOLD + flags |= FNM_CASEFOLD; +#endif + + data->in_section = fnmatch (section, data->name, flags) == 0; + return 0; +} + /* Configuration file parser. */ static void process_conf_file (const char *path, int exit_if_not_exist, process_line_fn process_line, - process_section_fn process_section) + process_section_fn process_section, + void *data) { static const char *whitespace = " \t\n\v"; FILE *fp; @@ -142,15 +245,15 @@ process_conf_file (const char *path, int exit_if_not_exist, exit (1); } - debug ("configuration file: section [%s]", line); + debug ("configuration file: section [%s]", &line[1]); if (!process_section) { error ("%s:%d: unexpected section header ([%s]) in file", - path, lineno, line); + path, lineno, &line[1]); exit (1); } - if (process_section (path, lineno, line) == -1) + if (process_section (path, lineno, &line[1], data) == -1) exit (1); } else { /* Key value */ @@ -159,14 +262,14 @@ process_conf_file (const char *path, int exit_if_not_exist, key = line; value = key_len < real_len ? &line[key_len+1] : NULL; if (value) { - value += strspn (line, whitespace); + value += strspn (value, whitespace); if (value[0] == '\0') value = NULL; } debug ("configuration file: key '%s', value '%s'", key, value); - if (process_line && process_line (path, lineno, key, value) == -1) + if (process_line && process_line (path, lineno, key, value, data) == -1) exit (1); } } diff --git a/hostinfod/hostinfo-protocol.pod b/hostinfod/hostinfo-protocol.pod index c50d6e4..eb9f40c 100644 --- a/hostinfod/hostinfo-protocol.pod +++ b/hostinfod/hostinfo-protocol.pod @@ -177,6 +177,135 @@ I<0x> to indicate hexadecimal literals. Boolean arguments should be sent as I or I. +=head2 AVAILCPUS + + AVAILCPUS + +=head3 Returns + +Number of available physical cores in the host. + +=head3 Example + + >>> AVAILCPUS + <<< 1.0 200 4 + +=head3 Description + +Returns the number of physical cores in the host that are available to +the virtualization system (ie. have not been disabled). See also +I which would in almost every case return the same number. + +=head2 CORESPERSOCKET + + CORESPERSOCKET + +=head3 Returns + +Number of cores per socket in the physical host. + +=head3 Example + + >>> CORESPERSOCKET + <<< 1.0 200 2 + +=head3 Description + +Returns the number of physical cores per socket in the host. + +=head2 MEMORY + + MEMORY + +=head3 Returns + +Amount of memory in host, in kilobytes. + +=head3 Example + + >>> MEMORY + <<< 1.0 200 2097022 + +=head3 Description + +Returns the total memory in the host, in kilobytes. + +=head2 MHZ + + MHZ + +=head3 Returns + +Speed of host cores in MHz. + +=head3 Example + + >>> MHZ + <<< 1.0 200 2047 + +=head3 Description + +Returns the clockspeed of host cores in MHz. + +=head2 MODEL + + MODEL + +=head3 Returns + +The host CPU model, a string such as C or C. + +=head3 Example + + >>> MODEL + <<< 1.0 200 x86_64 + +=head3 Description + +Returns the host CPU model. + +=head2 NODES + + NODES + +=head3 Returns + +The number of NUMA nodes in the host. + +=head3 Example + + >>> NODES + <<< 1.0 200 1 + +=head3 Description + +Returns the number of NUMA nodes in the host. If this is 1 +then host memory access is uniform. + +=head2 PHYSCPUS + + PHYSCPUS + +=head3 Returns + +The number of physical cores. + +=head3 Example + + >>> PHYSCPUS + <<< 1.0 200 4 + +=head3 Description + +Returns the number of physical cores available on the host. + +In some (highly unusual) situations, some cores might be +disabled. To get the number of cores available to do work, +use C. + +Note that it is common for the guest not to see all of the +physical CPUs (virtual CPUs E physical CPUs). + =head2 PING PING echodata @@ -230,6 +359,43 @@ I in the L manual page. =back +=head2 SOCKETSPERNODE + + SOCKETSPERNODE + +=head3 Returns + +The number of sockets on each node. + +=head3 Example + + >>> SOCKETSPERNODE + <<< 1.0 200 2 + +=head3 Description + +Returns the number CPU sockets in each NUMA node. + +=head2 THREADSPERCORE + + THREADSPERCORE + +=head3 Returns + +The number of hyperthreads per core. + +=head3 Example + + >>> THREADSPERCORE + <<< 1.0 200 1 + +=head3 Description + +If hyperthreading is enabled on the host, this returns +the number of threads on each real core. The numbers +returned by C and C are multiplied +accordingly. + diff --git a/hostinfod/hostinfod.h b/hostinfod/hostinfod.h index 98fcb63..8bb48e6 100644 --- a/hostinfod/hostinfod.h +++ b/hostinfod/hostinfod.h @@ -28,6 +28,8 @@ #include #include +#include + enum guest_state { guest_state_connecting, /* Connecting to socket. */ guest_state_request, /* Waiting or reading the request. */ @@ -59,6 +61,9 @@ struct guest_description { /* Increments every time guest does something bad, decremented once per min */ unsigned penalty; struct timespec last_penalty_decr; + + /* Hash records last time each command was run by this guest. */ + apr_hash_t *lasttime; }; enum arg_type { @@ -79,14 +84,16 @@ struct arg { extern const char *conf_file; extern char *socket_dir; extern char *guests_file; +extern char *libvirt_uri; +extern int libvirt_uri_set_on_cmdline; extern int verbose; extern int verbose_set_on_cmdline; extern int foreground; extern int foreground_set_on_cmdline; extern int messages_to_stderr; extern apr_pool_t *pool; /* pool for global memory allocation */ - extern void initialize (void); +extern struct timespec *diff_timespec (struct timespec *r, const struct timespec *a, const struct timespec *b); /* r = a - b */ /* error.c */ extern void init_syslog (void); @@ -107,6 +114,7 @@ extern void paprerror (apr_status_t r, const char *fs, ...) /* configuration.c */ extern void read_main_conf_file (void); +extern void check_guests_file (struct guest_description *hval, const char *cmd, double *interval, int *enabled); /* monitor_sockets.c */ extern int sockets_inotify_fd; @@ -141,4 +149,9 @@ typedef void (*command_fn) (struct guest_description *hval, const char *cmd, apr apr_hash_set (commands, #name, APR_HASH_KEY_STRING, cb); \ } +/* virt.c */ +extern void init_libvirt (void); +extern virConnectPtr conn; +extern virNodeInfo nodeinfo; + #endif /* HOSTINFOD_H */ diff --git a/hostinfod/main.c b/hostinfod/main.c index e9ca8ae..7c9601a 100644 --- a/hostinfod/main.c +++ b/hostinfod/main.c @@ -57,6 +57,9 @@ const char *conf_file = DEFAULT_CONF_FILE; char *socket_dir = NULL; char *guests_file = NULL; +char *libvirt_uri = NULL; +int libvirt_uri_set_on_cmdline = 0; + int verbose = 0; int verbose_set_on_cmdline = 0; int foreground = 0; @@ -91,6 +94,8 @@ usage (void) " --help Display full usage\n" " -c file | --config file\n" " Configuration file (default: %s)\n" + " -C uri | --connect uri\n" + " Set libvirt connection URI (default: NULL)\n" " -f | --foreground\n" " Run in the foreground (don't fork)\n" " -v Enable verbose messages (sent to syslog)\n", @@ -110,6 +115,7 @@ main (int argc, char *argv[]) { static const apr_getopt_option_t options[] = { { "config", 'c', TRUE, "configuration file" }, + { "connect", 'C', TRUE, "libvirt connection URI" }, { "foreground", 'f', FALSE, "run in foreground (don't fork)" }, { "verbose", 'v', FALSE, "enable verbose messages" }, { "help", '?', FALSE, "display help" }, @@ -146,6 +152,10 @@ main (int argc, char *argv[]) exit (1); } break; + case 'C': + libvirt_uri = optarg; + libvirt_uri_set_on_cmdline = 1; + break; case 'f': foreground = 1; foreground_set_on_cmdline = 1; @@ -169,6 +179,9 @@ main (int argc, char *argv[]) /* Read the config file. */ read_main_conf_file (); + /* Connect to libvirt. */ + init_libvirt (); + /* Monitor the socket directory. */ monitor_socket_dir (); @@ -457,6 +470,7 @@ guest_added (const char *sock_path, const char *name) hval->sock = sock; hval->request_max = 4096; hval->request = apr_palloc (hval->pool, hval->request_max); + hval->lasttime = apr_hash_make (hval->pool); /* Convert Unix fd into APR socket type. */ r = apr_os_sock_put (&hval->aprsock, &sock, hval->pool); @@ -530,7 +544,7 @@ guest_force_close (struct guest_description *hval) } /* Difference between two timespec structures (r = a - b) */ -static struct timespec * +struct timespec * diff_timespec (struct timespec *r, const struct timespec *a, const struct timespec *b) { diff --git a/hostinfod/nodeinfo.c b/hostinfod/nodeinfo.c new file mode 100644 index 0000000..793ea6e --- /dev/null +++ b/hostinfod/nodeinfo.c @@ -0,0 +1,168 @@ +/* virt-hostinfo + * Copyright (C) 2009 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +/* This file makes available all entries in the libvirt NodeInfo + * structure. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include + +#include "hostinfod.h" + +static void +model (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%s", nodeinfo.model); +} +REGISTER_COMMAND (model) + +static void +memory (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%lu", nodeinfo.memory); +} +REGISTER_COMMAND (memory) + +static void +availcpus (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%u", nodeinfo.cpus); +} +REGISTER_COMMAND (availcpus) + +static void +physcpus (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%u", VIR_NODEINFO_MAXCPUS (nodeinfo)); +} +REGISTER_COMMAND (physcpus) + +static void +mhz (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%u", nodeinfo.mhz); +} +REGISTER_COMMAND (mhz) + +static void +nodes (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%u", nodeinfo.nodes); +} +REGISTER_COMMAND (nodes) + +static void +socketspernode (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%u", nodeinfo.sockets); +} +REGISTER_COMMAND (socketspernode) + +static void +corespersocket (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%u", nodeinfo.cores); +} +REGISTER_COMMAND (corespersocket) + +static void +threadspercore (struct guest_description *hval, + const char *cmd, apr_array_header_t *args) +{ + if (get_args (args, "") == -1) { + warning ("%s: %s: wrong number or type of arguments", + hval->name, __func__); + send_error (hval, 400); + return; + } + + send_reply (hval, 200, "%u", nodeinfo.threads); +} +REGISTER_COMMAND (threadspercore) diff --git a/hostinfod/ping.c b/hostinfod/ping.c index 4a79f8c..daa9dcf 100644 --- a/hostinfod/ping.c +++ b/hostinfod/ping.c @@ -34,7 +34,8 @@ ping (struct guest_description *hval, size_t i, len; if (get_args (args, "s", &echodata) == -1) { - warning ("%s: %s: wrong number of type of arguments", hval->name, "ping"); + warning ("%s: %s: wrong number or type of arguments", hval->name, + __func__); send_error (hval, 400); return; } diff --git a/hostinfod/virt.c b/hostinfod/virt.c new file mode 100644 index 0000000..2628e6a --- /dev/null +++ b/hostinfod/virt.c @@ -0,0 +1,54 @@ +/* virt-hostinfo + * Copyright (C) 2009 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include +#include + +#include "hostinfod.h" + +virConnectPtr conn = NULL; +virNodeInfo nodeinfo; + +void +init_libvirt (void) +{ + virErrorPtr err; + + conn = virConnectOpenReadOnly (libvirt_uri); + if (!conn) { + err = virGetLastError (); + error ("could not connect to libvirt (code %d, domain %d): %s", + err->code, err->domain, err->message); + exit (1); + } + + if (virNodeGetInfo (conn, &nodeinfo) == -1) { + err = virConnGetLastError (conn); + error ("virGetNodeInfo failed (code %d, domain %d): %s", + err->code, err->domain, err->message); + exit (1); + } +} -- 1.8.3.1