2 * Copyright (C) 2013 Red Hat Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License along
15 * with this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
26 type ('a, 'b) alternative = Either of 'a | Or of 'b
28 let (//) = Filename.concat
29 let quote = Filename.quote
31 type goal_result_t = Goal_OK | Goal_failed of string
32 exception Goal_result of goal_result_t
34 let goal_failed msg = raise (Goal_result (Goal_failed msg))
40 for i = 0 to !depth-1 do prerr_string " " done;
42 Pervasives.flush Pervasives.stderr
47 if v then raise (Goal_result Goal_OK)
48 let target_all vs = target (List.fold_left (&&) true vs)
49 let target_exists vs = target (List.fold_left (||) false vs)
51 indent "require: %s\n" name;
53 let r = (try Either (f ()) with exn -> Or exn) in
59 let _enter_goal name = indent "enter goal: %s\n" name
60 let _leave_goal name = indent "leave goal: %s\n" name
62 type period_t = Seconds | Days | Months | Years
63 let seconds = (1, Seconds)
64 let sec = seconds and secs = seconds and second = seconds
65 let minutes = (60, Seconds)
66 let min = minutes and mins = minutes and minute = minutes
67 let hours = (3600, Seconds)
73 let months = (1, Months)
75 let years = (1, Years)
78 let periodic_jobs = ref []
80 (* Register a periodic job. *)
81 let every ?name i (j, t) f =
82 let period = i*j, t in (* 5 minutes -> ((5 * 60), Seconds) *)
83 periodic_jobs := (period, (name, f)) :: !periodic_jobs
85 (* [next_time t period] returns the earliest event of [period]
86 strictly after time [t].
88 Visualising periods as repeated events on a timeline, this
92 events: ---+---------+---------+---------+---------+---------+-----
96 Note that [period_t] events are not necessarily regular.
97 eg. The start of a month is not a fixed number of seconds
98 after the start of the previous month. 'Epoch' refers
99 to the Unix Epoch (ie. 1970-01-01 00:00:00 UTC).
101 If [period = i, Seconds i] then events are when
102 [t' mod i == 0] when t' is the number of seconds since
103 the Epoch. This returns the next t' > t.
105 If [period = i, Days] then events happen at
106 midnight UTC every [i] days since the Epoch.
107 This returns the next midnight > t.
109 If [period = i, Months] then events happen at
110 midnight UTC on the 1st day of the month every [i] months
111 since the Epoch. This returns midnight on the
112 1st day of the next month > t.
114 If [period = i, Years] then events happen at
115 midnight UTC on the 1st day of the year when
116 [(y - 1970) mod i == 0]. This returns midnight on the
117 1st day of the next year > t. *)
120 (* Round up 'a' to the next multiple of 'i'. *)
121 let round_up_float a i =
122 let r = mod_float a i in
123 if r = 0. then a +. i else a +. (i -. r)
126 if r = 0 then a + i else a + (i - r)
131 let i = float_of_int i in
137 (* Round 'tm' up to the first day of the next year. *)
138 let year = round_up tm.tm_year i in
139 let tm = { tm with tm_sec = 0; tm_min = 0; tm_hour = 0;
140 tm_mday = 1; tm_mon = 0; tm_year = year } in
144 let t = Date.from_unixfloat t in
145 let t0 = Date.make 1970 1 1 in
147 (* Number of whole days since Unix Epoch. *)
148 let nb_days = Date.Period.safe_nb_days (Date.sub t t0) in
150 let nb_days = round_up nb_days i in
151 let t' = Date.add t0 (Date.Period.day nb_days) in
155 (* Calculate number of whole months since Unix Epoch. *)
157 let months = 12 * (tm.tm_year - 70) + tm.tm_mon in
159 let months = round_up months i in
160 let t0 = Date.make 1970 1 1 in
161 let t' = Date.add t0 (Date.Period.month months) in
164 let file_exists = Sys.file_exists
166 let directory_exists path =
170 | Unix_error (ENOENT, _, _) -> None
171 | Unix_error (err, _, _) ->
172 let msg = sprintf "directory_exists: %s: %s" path (error_message err) in
175 | Some s -> s.st_kind = S_DIR
178 let file_newer_than f1 f2 =
182 | Unix_error (ENOENT, _, _) -> None
183 | Unix_error (err, _, _) ->
184 let msg = sprintf "file_newer_than: %s: %s" f (error_message err) in
187 let s1 = stat f1 and s2 = stat f2 in
193 let msg = sprintf "file_newer_than: %s: file does not exist" f2 in
196 s1.st_mtime >= s2.st_mtime
198 let more_recent objs srcs =
199 if not (List.for_all file_exists objs) then false
202 fun obj -> List.for_all (file_newer_than obj) srcs
207 (* http://stackoverflow.com/questions/12199059/how-to-check-if-an-url-exists-with-the-shell-and-probably-curl *)
209 sprintf "curl --output /dev/null --silent --head --fail %s" (quote url) in
210 match Sys.command cmd with
214 let msg = sprintf "curl error testing '%s': exit code %d, see curl(1)"
218 let file_contains_string filename str =
219 let cmd = sprintf "grep -q -F %s %s" (quote str) (quote filename) in
220 match Sys.command cmd with
224 let msg = sprintf "grep error testing for '%s' in '%s' (exit code %d)"
228 let url_contains_string url str =
229 let tmp = Filename.temp_file "goaljobsurl" "" in
231 sprintf "curl --output %s --silent --fail %s" (quote tmp) (quote url) in
232 (match Sys.command cmd with
235 let msg = sprintf "curl failed to download URL '%s'" url in
238 let msg = sprintf "curl error testing '%s': exit code %d, see curl(1)"
242 let r = file_contains_string tmp str in
246 (* Create a temporary directory. It is *not* deleted on exit. *)
248 let chan = open_in "/dev/urandom" in
249 let data = Bytes.create 16 in
250 really_input chan data 0 (Bytes.length data);
252 let data = Bytes.to_string data in
253 let data = Digest.to_hex (Digest.string data) in
254 let dir = Filename.temp_dir_name // sprintf "goaljobstmp%s" data in
258 (* Recursively remove directory. *)
260 let cmd = sprintf "rm -rf %s" (quote dir) in
261 ignore (Sys.command cmd)
263 let shell = ref "/bin/sh"
265 (* Used by sh, shout, shlines to handle the script and temporary dir. *)
266 let with_script ?(tmpdir = true) script f =
267 let dir = if tmpdir then Some (make_tmpdir ()) else None in
268 let script_file, chan =
271 let script_file = dir // "script.sh" in
272 let chan = open_out script_file in
274 | None -> Filename.open_temp_file "goaljobsscript" ".sh" in
275 chmod script_file 0o700;
276 fprintf chan "#!%s\n" !shell;
277 fprintf chan "set -e\n"; (* so that job exits on error *)
278 fprintf chan "set -x\n"; (* echo commands (must be last) *)
280 output_string chan script;
284 | Some dir -> sprintf "cd %s && exec %s" (quote dir) (quote script_file)
285 | None -> sprintf "exec %s" (quote script_file) in
286 let r = try Either (f cmd) with exn -> Or exn in
288 | Some dir -> rm_rf dir
293 | Or exn -> raise exn
297 with_script ?tmpdir script (
299 let r = Sys.command cmd in
301 let msg = sprintf "external command failed with code %d" r in
308 let do_shlines ?tmpdir script =
309 with_script ?tmpdir script (
311 let chan = open_process_in cmd in
312 let lines = ref [] in
314 let line = input_line chan in
315 eprintf "%s\n%!" line;
316 lines := line :: !lines;
319 (try loop () with End_of_file -> ());
320 match close_process_in chan with
321 | WEXITED 0 -> List.rev !lines
323 let msg = sprintf "external command failed with code %d" i in
326 let msg = sprintf "external command was killed by signal %d" i in
329 let msg = sprintf "external command was stopped by signal %d" i in
332 let shlines ?tmpdir fs = ksprintf (do_shlines ?tmpdir) fs
334 let do_shout ?tmpdir script =
335 let lines = do_shlines ?tmpdir script in
336 String.concat "\n" lines
337 let shout ?tmpdir fs = ksprintf (do_shout ?tmpdir) fs
340 val replace_substring : string -> string -> string -> string
343 let change_file_extension ext filename =
345 try String.rindex filename '.'
346 with Not_found -> String.length filename in
347 String.sub filename 0 i ^ "." ^ ext
350 val filter_file_extension : string -> string list -> string
353 (* Persistent memory is stored in $HOME/.goaljobs-memory. We have to
354 * lock this file each time we read or write because multiple concurrent
355 * jobs may access it at the same time.
357 * XXX Replace this with a more efficient and less fragile implementation.
360 let with_memory_locked ?(write = false) f =
361 let filename = getenv "HOME" // ".goaljobs-memory" in
362 let fd = openfile filename [O_RDWR; O_CREAT] 0o644 in
363 lockf fd (if write then F_LOCK else F_RLOCK) 0;
365 (* If the file is newly created with zero size, write an
368 if (fstat fd).st_size = 0 then (
369 let empty : (string, string) Hashtbl.t = Hashtbl.create 13 in
370 let chan = out_channel_of_descr fd in
371 output_value chan empty;
372 Pervasives.flush chan;
373 ignore (lseek fd 0 SEEK_SET)
376 (* Run the function. *)
377 let r = try Either (f fd) with exn -> Or exn in
382 | Or exn -> raise exn
384 let memory_exists key =
387 let chan = in_channel_of_descr fd in
388 let memory : (string, string) Hashtbl.t = input_value chan in
389 Hashtbl.mem memory key
395 let chan = in_channel_of_descr fd in
396 let memory : (string, string) Hashtbl.t = input_value chan in
397 try Some (Hashtbl.find memory key) with Not_found -> None
400 let memory_set key value =
401 with_memory_locked ~write:true (
403 let chan = in_channel_of_descr fd in
404 let memory : (string, string) Hashtbl.t = input_value chan in
405 Hashtbl.replace memory key value;
406 let chan = out_channel_of_descr fd in
408 output_value chan memory;
409 Pervasives.flush chan;
412 let memory_delete key =
413 with_memory_locked ~write:true (
415 let chan = in_channel_of_descr fd in
416 let memory : (string, string) Hashtbl.t = input_value chan in
417 Hashtbl.remove memory key;
418 let chan = out_channel_of_descr fd in
420 output_value chan memory;
421 Pervasives.flush chan;
427 let chan = in_channel_of_descr fd in
428 let memory : (string, string) Hashtbl.t = input_value chan in
429 Hashtbl.fold (fun key value xs -> (key, value) :: xs) memory []
432 let published_goals = ref []
433 let publish name fn = published_goals := (name, fn) :: !published_goals
435 try Some (List.assoc name !published_goals) with Not_found -> None
437 let log_program_output () =
438 let filename = Filename.temp_file "goaljobslog" ".txt" in
439 let cmd = "tee " ^ quote filename in
440 let chan = open_process_out cmd in
441 let fd = descr_of_out_channel chan in
446 let mailto ?from ~subject ?(attach = []) to_ =
447 let cmd = ref (sprintf "%s -s %s" mailx (quote subject)) in
450 | Some f -> cmd := !cmd ^ " -r " ^ quote f
453 fun a -> cmd := !cmd ^ " -a " ^ quote a
455 cmd := !cmd ^ " " ^ quote to_;
456 if Sys.command !cmd <> 0 then
457 goal_failed "mailto: could not send email"
459 let goal_file_exists filename =
460 if not (file_exists filename) then (
461 let msg = sprintf "file '%s' required but not found" filename in
464 let goal_directory_exists path =
465 if not (directory_exists path) then (
466 let msg = sprintf "directory '%s' required but not found" path in
469 let goal_file_newer_than f1 f2 =
470 if not (file_newer_than f1 f2) then (
471 let msg = sprintf "file %s is required to be newer than %s" f1 f2 in
474 let goal_more_recent objs srcs =
475 if not (more_recent objs srcs) then (
476 let msg = sprintf "object(s) %s are required to be newer than source(s) %s"
477 (String.concat " " objs) (String.concat " " srcs) in
480 let goal_url_exists url =
481 if not (url_exists url) then (
482 let msg = sprintf "url_exists: URL '%s' required but does not exist" url in
485 let goal_file_contains_string filename str =
486 if not (file_contains_string filename str) then (
487 let msg = sprintf "file_contains_string: file '%s' is required to contain string '%s'" filename str in
490 let goal_url_contains_string url str =
491 if not (url_contains_string url str) then (
492 let msg = sprintf "url_contains_string: URL '%s' is required to contain string '%s'" url str in
495 let goal_memory_exists k =
496 if not (memory_exists k) then (
497 let msg = sprintf "memory_exists: key '%s' required but does not exist" k in
504 | Goal_result (Goal_failed msg) ->
505 prerr_endline ("error: " ^ msg);
508 prerr_endline (Printexc.to_string exn);
511 (* Run the program. *)
513 let prog = Sys.executable_name in
514 let prog = Filename.basename prog in
516 (* Save the current working directory when the program started. *)
517 putenv "builddir" (getcwd ());
521 let display_version () =
522 printf "%s %s\n" package_name package_version;
527 let names = !published_goals in
528 let names = List.map fst names in
529 let names = List.sort compare names in
530 List.iter print_endline names;
534 let argspec = Arg.align [
535 "--goals", Arg.Unit list_goals, " List all goals";
536 "-l", Arg.Unit list_goals, " List all goals";
537 "-V", Arg.Unit display_version, " Display version number and exit";
538 "--version", Arg.Unit display_version, " Display version number and exit";
540 let anon_fun str = args := str :: !args in
541 let usage_msg = sprintf "\
542 %s: a script generated by goaljobs
544 List all goals: %s -l
545 Run a single goal like this: %s <name-of-goal> [<goal-args ...>]
547 For more information see the goaljobs(1) man page.
552 Arg.parse argspec anon_fun usage_msg;
554 let args = List.rev !args in
556 (* Was a goal named on the command line? *)
559 (match get_goal name with
561 exit (if guard fn args then 0 else 1)
563 eprintf "error: no goal called '%s' was found.\n" name;
564 eprintf "Use %s -l to list all published goals in this script.\n" name;
568 (* If periodic jobs exist, fall through. *)
569 if !periodic_jobs = [] then (
570 (* Does a published 'all' goal exist? *)
571 match get_goal "all" with
573 exit (if guard fn [] then 0 else 1)
575 (* No published 'all' goal. *)
576 eprintf "error: no goal called 'all' was found.\n";
581 assert (!periodic_jobs <> []);
583 (* Run the periodic jobs. Note these run forever, or until killed. *)
585 (* Find the next job to run. *)
587 let jobs = List.map (
588 fun (period, (_, _ as name_f)) ->
589 next_time now period, name_f
591 let jobs = List.sort (fun (t1,_) (t2,_) -> compare t1 t2) jobs in
593 (* Find all jobs that have the same next time.
594 * XXX When we can handle parallel jobs we can do better here,
595 * but until them run all the ones which have the same time
598 let next_t = int_of_float (fst (List.hd jobs)) in
599 let jobs = List.filter (fun (t, _) -> int_of_float t = next_t) jobs in
601 (* Run next job(s) after waiting for the appropriate amount of time. *)
602 let seconds = next_t - int_of_float now in
603 eprintf "next job will run in %s\n%!" (printable_seconds seconds);
607 fun (_, (name, f)) ->
608 eprintf "running job: %s\n%!"
609 (match name with Some name -> name | None -> "[unnamed]");
614 and printable_seconds s =
615 if s < 60 then sprintf "%d seconds" s
616 else if s < 6000 then sprintf "%d minutes, %d seconds" (s/60) (s mod 60)
617 else if s < 86400 then sprintf "%d hours, %d minutes" (s/3600) (s/60)
618 else sprintf "about %d days" (s/86400)