# Mark objects as 'ancient' so they are taken out of the OCaml heap.
-# $Id: Makefile,v 1.6 2006-09-27 16:01:47 rich Exp $
+# $Id: Makefile,v 1.7 2006-10-06 12:26:31 rich Exp $
include Makefile.config
OCAMLDOCFLAGS := -html -stars -sort $(OCAMLCPACKAGES)
-ifeq ($(TEST_WEBLOGS),1)
-# For testing with large amount of weblogs data.
-OCAMLCPACKAGES := -package calendar,pcre,extlib -I ../../freeware/weblogs
-OCAMLCLIBS := -linkpkg weblogs.cma
-OCAMLOPTPACKAGES := $(OCAMLCPACKAGES)
-OCAMLOPTLIBS := -linkpkg weblogs.cmxa
-endif
-
TARGETS := mmalloc ancient.cma ancient.cmxa META \
test_ancient.opt test_ancient_shared.opt
-ifeq ($(TEST_WEBLOGS),1)
-TARGETS += test_ancient_weblogs.opt
-endif
-
all: $(TARGETS)
ancient.cma: ancient.cmo ancient_c.o
LIBRARY_PATH=.:$$LIBRARY_PATH \
ocamlfind ocamlopt $(OCAMLOPTFLAGS) $(OCAMLOPTPACKAGES) $(OCAMLOPTLIBS) -o $@ $^
-ifeq ($(TEST_WEBLOGS),1)
-test_ancient_weblogs.opt: ancient.cmxa test_ancient_weblogs.cmx
- LIBRARY_PATH=.:$$LIBRARY_PATH \
- ocamlfind ocamlopt $(OCAMLOPTFLAGS) $(OCAMLOPTPACKAGES) $(OCAMLOPTLIBS) -o $@ $^
-endif
-
# Build the mmalloc library.
mmalloc:
install:
rm -rf $(DESTDIR)$(OCAMLLIBDIR)/ancient
- install -c -m 0755 -d $(DESTDIR)$(OCAMLLIBDIR)/weblogs
+ install -c -m 0755 -d $(DESTDIR)$(OCAMLLIBDIR)/ancient
install -c -m 0644 *.cmi *.mli *.cma *.cmxa *.a META \
$(DESTDIR)$(OCAMLLIBDIR)/ancient
+++ /dev/null
-(* Load in large weblogs and see if they can still be used.
- * $Id: test_ancient_weblogs.ml,v 1.4 2006-10-06 12:25:20 rich Exp $
- *)
-
-open Printf
-
-open ExtList
-
-let gc_stats = true (* If true, print GC stats before processing each day. *)
-
-let (//) = Filename.concat
-
-let rec range a b =
- if a > b then []
- else a :: range (succ a) b
-
-(* Cartesian join of two lists. *)
-let cartesian xs ys =
- List.flatten (
- List.map (
- fun x ->
- List.map (
- fun y -> x, y
- ) ys
- ) xs
- )
-
-let file_readable filename =
- try Unix.access filename [Unix.R_OK]; true
- with Unix.Unix_error _ -> false
-
-(* Suppress warning messages. *)
-let () = Weblogs.quiet := true
-
-let gc_compact () =
- eprintf "compacting ... %!";
- Gc.compact ();
- if gc_stats then (
- let stat = Gc.stat () in
- let live_words = stat.Gc.live_words in
- eprintf "live words = %d (%d MB)\n%!"
- live_words (live_words * 8 / 1024 / 1024)
- )
-
-(* Find the list of files. Some which should exist don't, so
- * warnings about those so we can chase up.
- *)
-let files =
- let dir = "/home/rich/oversized-logfiles/perrys" in
- let drivers =
- [ "burns"; "gronholm"; "rohrl"; "sainz"; "solberg"; "vatanen" ] in
- let dates = range 1 31 in
- let dates = List.map (fun day -> sprintf "200608%02d" day) dates in
- let files = cartesian drivers dates in
- let files =
- List.map (fun (driver, date) ->
- sprintf "%s-perrys-access.log.%s.gz" driver date) files in
- let files =
- List.filter_map (
- fun filename ->
- let path = dir // filename in
- if not (file_readable path) then (
- prerr_endline ("warning: " ^ filename ^ " not found - ignored");
- None
- ) else (
- Some path
- )
- ) files in
-
- eprintf "number of files = %d\n%!" (List.length files);
-
- files
-
-(*
-(* XXX Linux/AMD64-specific hack to avoid bad mmap(2) allocation. *)
-let baseaddr = Nativeint.of_string "0x440000000000"
-
-let md =
- let fd =
- Unix.openfile "test_ancient_weblogs.data"
- [Unix.O_RDWR; Unix.O_CREAT; Unix.O_TRUNC] 0o644 in
- Ancient.attach fd baseaddr
-
-(* Load each file into memory and make it ancient. *)
-let () =
- List.iteri (
- fun key filename ->
- let () =
- let basename = Filename.basename filename in
- eprintf "Importing logfile %s\n%!" basename;
- let rows = Weblogs.import_file filename in
- ignore (Ancient.share md key rows) in
- gc_compact ()
- ) files;
-
- Ancient.detach md
-*)
-
-let () =
- let fd = Unix.openfile "test_ancient_weblogs.data" [Unix.O_RDWR] 0o644 in
- let md = Ancient.attach fd 0n in
-
- eprintf "Flattening ...\n%!";
-
- (* Concatenate all the logs together. *)
- let rows =
- List.flatten (
- List.mapi (
- fun key _ ->
- let rows : Weblogs.t Ancient.ancient = Ancient.get md key in
- let rows = Ancient.follow rows in
- rows
- ) files
- ) in
-
- eprintf "After flattening: %!";
- gc_compact ();
-
- (* Detect visitors. Save to key 1023 in the file. The detect_visitors
- * function sorts each visitor.
- *)
- let visitors = Weblogs.detect_visitors rows in
- ignore (Ancient.share md 1023 visitors);
-
- eprintf "After detecting visitors: %!";
- gc_compact ();
-
- Ancient.detach md