From 153276d53f879786956bd7c2a4d3b97ef13c9adc Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Sun, 15 Sep 2013 21:36:01 +0100 Subject: [PATCH] Initial revision. --- .gitignore | 1 + Makefile.am | 70 +++++++++++++++++ NOTES | 80 +++++++++++++++++++ configure.ac | 91 +++++++++++++++++++++ examples/Makefile.am | 17 ++++ goaljobs.ml | 96 +++++++++++++++++++++++ goaljobs.mli | 218 +++++++++++++++++++++++++++++++++++++++++++++++++++ tests/Makefile.am | 17 ++++ 8 files changed, 590 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile.am create mode 100644 NOTES create mode 100644 configure.ac create mode 100644 examples/Makefile.am create mode 100644 goaljobs.ml create mode 100644 goaljobs.mli create mode 100644 tests/Makefile.am diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b25c15b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..11997d7 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,70 @@ +# goaljobs +# Copyright (C) 2013 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +ACLOCAL_AMFLAGS = -I m4 + +EXTRA_DIST = \ + COPYING \ + NOTES \ + README \ + $(sources) + +CLEANFILES = *~ + +OCAMLCFLAGS = -g -package unix +OCAMLOPTFLAGS = $(OCAMLCFLAGS) + +SUBDIRS = . examples tests + +sources = \ + test.ml \ + goaljobs.ml \ + goaljobs.mli + +noinst_SCRIPTS = test + +test: goaljobs.cmx + $(OCAMLFIND) ocamlopt $(OCAMLOPTFLAGS) $< -o $@ + +# Dependencies. + +%.cmi: %.mli + $(OCAMLFIND) ocamlc $(OCAMLCFLAGS) -c $< -o $@ +%.cmo: %.ml + $(OCAMLFIND) ocamlc $(OCAMLCFLAGS) -c $< -o $@ +%.cmx: %.ml + $(OCAMLFIND) ocamlopt $(OCAMLOPTFLAGS) -c $< -o $@ + +depend: .depend + +.depend: $(wildcard $(abs_srcdir)/*.mli) $(wildcard $(abs_srcdir)/*.ml) + rm -f $@ $@-t + $(OCAMLFIND) ocamldep -I $(abs_srcdir) $^ | \ + $(SED) 's/ *$$//' | \ + $(SED) -e :a -e '/ *\\$$/N; s/ *\\\n */ /; ta' | \ + $(SED) -e 's,$(abs_srcdir)/,$(builddir)/,g' | \ + sort > $@-t + mv $@-t $@ + +-include .depend + +SUFFIXES = .cmo .cmi .cmx .ml .mli .mll .mly + +# License check. + +licensecheck: + licensecheck $$(git ls-files) diff --git a/NOTES b/NOTES new file mode 100644 index 0000000..14afc95 --- /dev/null +++ b/NOTES @@ -0,0 +1,80 @@ +Like 'make' except: + - Predicates can be based on arbitrary expressions, not just + "file X is older than file Y". + - Rules are more flexible and encourage structuring and reuse + through functions. + - Goals can be parameterized. + - Program can run continuously to implement business rules. + +Differences from 'whenjobs': + - Goals instead of variables. + - Persistent (across session) variables exist, but are not central. + - Doesn't use <<..>> for shell scripts (has a function 'sh' instead). + +Similarities to 'whenjobs': + - Each shell script runs in its own temporary directory. + +Example Makefile rule and translation to goaljobs language: + + %.o: %.c + cc -c $< -o $@ + + let goal compile c_file = + require (file_exists c_file); + let o_file = replace_substring ".c" ".o" c_file in + target (file_exists o_file && file_newer o_file c_file); + sh "cc -c %s -o %s" c_file o_file + +Example program: + + let package = "foo" + + let rec goal website_updated version = + let tarfile = sprintf "%s-%s.tar.gz" package version in + let tarpath = getenv "HOME" // "html" // tarfile in + let url = sprintf "http://example.com/%s" tarfile in + + target (url_exists url); + + require (tarball_exists version); + require (tarball_tested version); + + sh "rsync %s example.com:/html/" tarpath + + and goal tarball_tested version = + let tarfile = sprintf "%s-%s.tar.gz" package version in + let tarpath = getenv "HOME" // "html" // tarfile in + let memkey = package ^ "_tested_" ^ version in + + target (memory_exists memkey); + + require (tarball_exists version); + + sh " + tar zxf %s + cd %s-%s + ./configure + make + make check + " tarpath package version; + + memory_set memkey "1" + + and goal tarball_exists version = + let tarpath = getenv "HOME" // "html" // tarfile in + target (file_exists tarpath); + sh " + cd $HOME/repos/%s + git fetch + git archive --prefix %s-%s/ v%s | gzip > %s-t + mv %s-t %s + " package package version version tarpath tarpath tarpath + + every 1 hour = + let version = shout " + cd $HOME/repos/%s + git fetch + git describe --tags --abbrev=0 --match='v*' + " package in + require (website_updated version) + diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..1f3189b --- /dev/null +++ b/configure.ac @@ -0,0 +1,91 @@ +# goaljobs +# Copyright (C) 2013 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +AC_INIT([goaljobs],[0.1]) +AM_INIT_AUTOMAKE([foreign]) + +AC_CONFIG_MACRO_DIR([m4]) + +dnl Allow all GNU/Linux functions. +dnl autoconf complains unless this is very early in the file. +AC_USE_SYSTEM_EXTENSIONS + +AC_PROG_LIBTOOL + +dnl Check for basic C environment. +AC_PROG_CC_STDC +AC_PROG_INSTALL +AC_PROG_CPP + +AC_C_PROTOTYPES +test "x$U" != "x" && AC_MSG_ERROR([Compiler not ANSI compliant]) + +AM_PROG_CC_C_O + +AC_ARG_ENABLE([gcc-warnings], + [AS_HELP_STRING([--enable-gcc-warnings], + [turn on lots of GCC warnings (for developers)])], + [case $enableval in + yes|no) ;; + *) AC_MSG_ERROR([bad value $enableval for gcc-warnings option]) ;; + esac + gcc_warnings=$enableval], + [gcc_warnings=no] +) + +if test "$gcc_warnings" = yes; then + # XXX With gnulib we can improve this in future. + WARN_CFLAGS="-Wall" + AC_SUBST([WARN_CFLAGS]) + WERROR_CFLAGS="-Werror" + AC_SUBST([WERROR_CFLAGS]) +fi + +dnl Check support for 64 bit file offsets. +AC_SYS_LARGEFILE + +dnl OCaml compiler. +AC_PROG_OCAML +if test "$OCAMLC" = "no"; then + AC_MSG_ERROR([You must install the OCaml compiler]) +fi + +AM_CONDITIONAL([HAVE_OCAMLOPT], [test "x$OCAMLOPT" != "xno"]) + +dnl Camlp4 is required. +AC_PROG_CAMLP4 +if test "x$CAMLP4" = "xno"; then + AC_MSG_ERROR([You must install camlp4 (the OCaml macro preprocessor)]) +fi + +dnl OCaml findlib ("ocamlfind") is required. +AC_PROG_FINDLIB +if test "x$OCAMLFIND" = "xno"; then + AC_MSG_ERROR([You must install OCaml findlib (the ocamlfind command)]) +fi + +dnl Check for POD (for manual pages). +AC_CHECK_PROG(PERLDOC,perldoc,perldoc) +if test "x$PERLDOC" = "x"; then + AC_MSG_ERROR([You must install the perldoc program]) +fi + +AC_CONFIG_HEADERS([config.h]) +AC_CONFIG_FILES([Makefile + examples/Makefile + tests/Makefile]) +AC_OUTPUT diff --git a/examples/Makefile.am b/examples/Makefile.am new file mode 100644 index 0000000..9445514 --- /dev/null +++ b/examples/Makefile.am @@ -0,0 +1,17 @@ +# goaljobs +# Copyright (C) 2013 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + diff --git a/goaljobs.ml b/goaljobs.ml new file mode 100644 index 0000000..45422d2 --- /dev/null +++ b/goaljobs.ml @@ -0,0 +1,96 @@ +(* goaljobs + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Unix +open Printf + +type goal_result_t = Goal_OK | Goal_failed of string +exception Goal_result of goal_result_t + +let goal_failed msg = raise (Goal_result (Goal_failed msg)) + +let target v = + if v then raise (Goal_result Goal_OK) +let require = function + | Goal_OK -> () + | r -> raise (Goal_result r) + +let file_exists = Sys.file_exists + +let file_newer_than f1 f2 = + let stat f = + try Some (stat f) + with + | Unix_error (ENOENT, _, _) -> None + | Unix_error (err, _, _) -> + let msg = sprintf "file_newer_than: %s: %s" f (error_message err) in + goal_failed msg + in + let s1 = stat f1 and s2 = stat f2 in + match s1 with + | None -> () + | Some s1 -> + match s2 with + | None -> + let msg = sprintf "file_newer_than: %s: file does not exist" f2 in + goal_failed msg + | Some s2 -> + s1.st_mtime >= s2.st_mtime + +let url_exists url = goal_failed "url_exists not implemented!" + +let sh fs = + let do_sh cmd = + print_endline cmd; + let cmd = "set -e\n\n" ^ cmd in + let r = System.command cmd in + if r <> 0 then ( + let msg = sprintf "sh: external command failed with code %d" r in + goal_failed msg + ) + in + ksprintf do_sh fs + +(* +val shout : ('a, unit, string) format -> 'a +val shlines : ('a, unit, string) format -> 'a + +val shell : string ref +*) + +(* +val replace_substring : string -> string -> string -> string +val change_file_extension : string -> string -> string +val filter_file_extension : string -> string list -> string +*) + +let goal_file_exists filename = + if not (file_exists filename) then ( + let msg = sprintf "file_exists: %s: file not found" filename in + goal_failed msg + ) +let goal_file_newer_than f1 f2 = + if not (file_newer_than f1 f2) then ( + let msg = sprintf "file %s is not newer than %s" f1 f2 in + goal_failed msg + ) +let goal_url_exists url = + if not (url_exists url) then ( + let msg = sprintf "url_exists: %s: URL does not exist" url in + goal_failed msg + ) diff --git a/goaljobs.mli b/goaljobs.mli new file mode 100644 index 0000000..6839108 --- /dev/null +++ b/goaljobs.mli @@ -0,0 +1,218 @@ +(* goaljobs + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** {1 Goaljobs library of useful helper functions.} *) + +(** {2 Targets and requires} + + These are used to write goals. + + Normally you write a goal with one or more [target]s and + zero or more [require]s, as the examples below should make + clear. + + In the first example, there are two targets: that [o_file] (object) + exists, and that it is newer than [c_file] (source). The rule + meets that target by running the C compiler ([cc]) which, if it + succeeds, will ensure that the object file exists and is newer + than the source file. + + {v + let goal compiled c_file = + let o_file = change_file_extension "o" c_file in + target (file_exists o_file); + target (file_newer_than o_file c_file); + + sh "cc -c %s -o %s" c_file o_file + } + + In the second example, the rule requires that several files + have been compiled ([require (compiled ...)] + before it can link the final program: + + {v + let goal built program sources = + target (file_exists program); + target (file_newer_than program sources); + + List.iter (fun s -> require (compiled s)) sources; + + let objects = List.map (change_file_extension "o") sources in + sh "cc %s -o %s" (String.concat " " objects) program + } + +*) + +val target : bool -> unit + (** [target] {i predicate} defines the target condition that will + be met once the current rule has run. + + Goaljobs is much more flexible than [make]. In [make] only a + single type of target is possible. The following are roughly + equivalent: + + {v + foo.o: foo.c + ... + + let goal compiled () = + target (file_exists "foo.o"); + target (file_newer_than "foo.o" "foo.c"); + ... + } + + Targets in goaljobs can be any arbitrary expression, and you + can have any number of different targets. + + Almost every rule should have one or more targets, which should + accurately state the outcome once the rule has been run + + Normally you put the target(s) early on in the rule, before any + running code and before any [require]s. This is not a + hard-and-fast rule and it is not enforced, but doing it will + ensure the rule runs most efficiently since if the target is met + already then the rest of the rule doesn't run. *) + +val require : unit -> unit + (** [require] {!goal} defines the requirements of this rule, that + is, other goals that have to be met before this rule is able to run. + + In terms of [make], [require]s are roughly equivalent to the + right hand side after the [:], but in goaljobs the requirements + can be much richer than simply "that file must exist". + + Some simple rules don't need any [require]s. Unlike with [make], + the requirements of a rule can be placed anywhere within the + rule, as long as you put them before they are needed. *) + +(** {2 File and URL testing} + + Various functions to test the existence of files, URLs. +*) + +val file_exists : string -> bool + (** Return true if the named file exists. + + This function also exists as a goal. Writing: + {v require (file_exists "somefile");} + will die unless ["somefile"] exists. *) + +val file_newer_than : string -> string -> bool + (** [file_newer_than file_a file_b] returns true if [file_a] is + newer than [file_b]. Note that if [file_a] does not exist, it + returns false. If [file_b] does not exist, it is an error. *) + +val url_exists : string -> bool + (** The URL is tested to see if it exists. + + This function also exists as a goal. Writing: + {v require (url_exists "http://example.com");} + will die unless the given URL exists. *) + +(** {2 Shell} + + Call out to the Unix shell. [/bin/sh] is used unless you set + {!shell} to some other value. Note that the environment variable + [SHELL] is {i not} used. + + {!sh}, {!shout}, {!shlines} work like [printf]. ie. You can + substitute variables using [%s], [%d] and so on. For example: + + {v + sh "rsync foo-%s.tar.gz example.com:/html/" version + } + + Each invocation of {!sh} (etc) is a single shell (this is slightly + different from how [make] works). For example: + + {v + sh " + package=foo-%s + tarball=$package.tar.gz + cp $HOME/$tarball . + tar zxf $tarball + cd $package + ./configure + make + " version + } + + The shell error mode is set such that if any single command + returns an error then the {!sh} function as a whole exits with + an error. Write: + {v command ||: } + to ignore the result of a command. + + Each shell runs in a new temporary directory. The temporary directory + and all its contents is deleted after the shell exits. If you + want to save any data, [cd] somewhere. For example you could start + the command sequence with: + {v cd $HOME/data/ } +*) + +val sh : ('a, unit, string, unit) format4 -> 'a -> unit + (** Run the command(s). *) + +(* +val shout : ('a, unit, string) format -> 'a + (** Run the command(s). Anything printed on stdout is returned + as a single string (the trailing [\n] character, if any, + is not returned). *) + +val shlines : ('a, unit, string) format -> 'a + (** Run the command(s). Any lines printed to stdout are returned + as a list of strings. Trailing [\n] characters not returned. *) + +val shell : string ref + (** Set this variable to override the default shell ([/bin/sh]). *) +*) + +(** {2 String functions} + + Most string functions are provided by the OCaml standard + library (see the module [String]). For convenience some + extra functions are provided here. *) + +val replace_substring : string -> string -> string -> string + (** [replace_substring patt repl string] replaces all occurrences + of [patt] with [repl] in [string]. *) + +val change_file_extension : string -> string -> string + (** [change_file_extension ext filename] changes the file extension + of [filename] to [.ext]. For example + [change_file_extension "o" "main.c"] returns ["main.o"]. + If the original filename has no extension, this function + adds the extension. *) + +val filter_file_extension : string -> string list -> string + (** [filter_file_extension ext filenames] returns only those + filenames in the list which have the given file extension. + For example [filter_file_extension "o" ["foo.c"; "bar.o"]] + would return [["bar.o"]] (a single element list). *) + +(**/**) + +(* Goal versions of some common functions. You are using these + * versions when you write something like: + * require (file_exists "foo"); + * They work the same way as the regular function, except they die + * if the predicate returns false. + *) +val goal_file_exists : string -> unit +val goal_file_newer_than : string -> string -> unit +val goal_url_exists : string -> unit diff --git a/tests/Makefile.am b/tests/Makefile.am new file mode 100644 index 0000000..9445514 --- /dev/null +++ b/tests/Makefile.am @@ -0,0 +1,17 @@ +# goaljobs +# Copyright (C) 2013 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + -- 1.8.3.1