X-Git-Url: http://git.annexia.org/?a=blobdiff_plain;f=scripts%2Fpage.ml;h=30b72b6d0ad5a2b520a4a5399797a327f05ee092;hb=c7a57233daadec4f174176a474fc7b5018cdf986;hp=0f85ca800320333d93a430c7a9a325a22b968dd8;hpb=1f125fd7a6794f352f30de71a3905d5356a07008;p=cocanwiki.git diff --git a/scripts/page.ml b/scripts/page.ml index 0f85ca8..30b72b6 100644 --- a/scripts/page.ml +++ b/scripts/page.ml @@ -1,7 +1,22 @@ -(* COCANWIKI scripts. +(* COCANWIKI - a wiki written in Objective CAML. * Written by Richard W.M. Jones . * Copyright (C) 2004 Merjis Ltd. - * $Id: page.ml,v 1.10 2004/09/09 09:35:33 rich Exp $ + * $Id: page.ml,v 1.51 2006/08/03 13:52:58 rich Exp $ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. *) open Apache @@ -10,115 +25,391 @@ open Cgi open Printf open ExtString +open ExtList open Cocanwiki open Cocanwiki_template open Cocanwiki_ok open Cocanwiki_date +open Cocanwiki_server_settings +open Cocanwiki_links +open Cocanwiki_extensions +open Cocanwiki_strings -(* Maximum level of redirection. *) -let max_redirect = 4 - -type fp_status = FPOK of int * string * string * Dbi.datetime * bool - | FPRedirect of string +type fp_status = FPOK of int32 * string * string * Calendar.t * bool + | FPInternalRedirect of string + | FPExternalRedirect of string | FPNotFound -let run r (q : cgi) (dbh : Dbi.connection) hostid {edit_anon=edit_anon} user = - let template_page = get_template dbh hostid "page.html" in - let template_404 = get_template dbh hostid "page_404.html" in +(* Referer strings which help us decide if the user came from + * a search engine and highlight terms in the page appropriately. + *) +let search_engines = [ + Pcre.regexp "^http://.*google\\.", [ "q"; "as_q"; "as_epq"; "as_oq" ]; + Pcre.regexp "^http://.*yahoo\\.", [ "p" ]; + Pcre.regexp "^http://.*msn\\.", [ "q"; "MT" ] +] +let split_words = Pcre.regexp "\\W+" + +let split_qs_re = Pcre.regexp "\\?" + +let xhtml_re = Pcre.regexp "<.*?>|[^<>]+" +let run r (q : cgi) dbh hostid + ({ edit_anon = edit_anon; view_anon = view_anon } as host) + user = let page = q#param "page" in let page = if page = "" then "index" else page in - (* Host-specific CSS? *) - let sth = dbh#prepare_cached "select css is not null from hosts - where id = ?" in - sth#execute [`Int hostid]; - let has_host_css = - match sth#fetch1 () with - | [ `Bool has_host_css ] -> has_host_css + (* The main "page" template is split in two to improve the speed of + * delivery of the page. The very first part ("page_header.html") + * contains the page , crucially including all the links to the + * stylesheets. We send this first and flush it out to the client so + * that the client can begin requesting stylesheets, background images + * and so on. After this we compose the main page ("page.html") and + * send it out second. + *) + + let template_page_header = + get_template ~page dbh hostid "page_header.html" in + let template_page = get_template ~page dbh hostid "page.html" in + + (* This is the simpler template for 404 pages. *) + let template_404_header = get_template dbh hostid "page_404_header.html" in + let template_404 = get_template dbh hostid "page_404.html" in + + (* Host-specific fields. *) + let rows = + PGSQL(dbh) + "select feedback_email is not null, + mailing_list, navigation + from hosts where id = $hostid" in + let has_feedback_email, mailing_list, navigation = + match rows with + | [Some has_feedback_email, mailing_list, navigation] -> + has_feedback_email, mailing_list, navigation | _ -> assert false in - (* Can the user edit? Manage users? *) - let can_edit = can_edit edit_anon user in - let can_manage_users = can_manage_users user in + (* User permissions. *) + let can_edit = can_edit host user in + let can_manage_users = can_manage_users host user in + + (* Do we have a stats page set up? *) + let has_stats = server_settings_stats_page dbh <> None in + + (* Given the referer string, return the list of search terms. If none + * can be found, then throws Not_found. + *) + let search_terms_from_referer referer = + let _, argnames = + List.find (fun (rex, _) -> Pcre.pmatch ~rex referer) search_engines in + let url, qs = + match Pcre.split ~rex:split_qs_re ~max:2 referer with + | [url] | [url;""] -> url, "" + | [url;qs] -> url, qs + | _ -> assert false in + let args = Cgi_args.parse qs in + let argname = + List.find (fun argname -> List.mem_assoc argname args) argnames in + let search_string = List.assoc argname args in + Pcre.split ~rex:split_words search_string + in + + (* Given a full page of XHTML, highlight search terms found in the + * part of the page. + *) + let highlight_search_terms xhtml search_terms span_class = + (* Split the original XHTML into strings and tags. For example if + * the original string is: "This is some bold text.
", then + * after this step we will have the following list: + * [ "This is some "; ""; "bold"; ""; " text."; "
" ] + *) + let xhtml = Pcre.extract_all ~rex:xhtml_re xhtml in + let xhtml = Array.to_list xhtml in + let xhtml = List.map (fun matches -> matches.(0)) xhtml in + + (* Find the ... tags. We only want to apply + * highlighting to tags within this area. + *) + let rec list_split f acc = function + | [] -> List.rev acc, [] + | ((x :: _) as xs) when f x -> List.rev acc, xs + | x :: xs -> + let acc = x :: acc in + list_split f acc xs + in + let head, body = + list_split (fun str -> String.starts_with str "") [] body in + (* NB: Hopefully, xhtml = head @ body @ tail. *) + + (* The search terms are a list of simple words. Turn into a big + * regular expression, because we want to substitute for each. We + * end up with a regexp like '(word1|word2|word3)'. + *) + let rex = + Pcre.regexp ~flags:[`CASELESS] + ("(" ^ String.concat "|" search_terms ^ ")") in + + (* Do the substitution, but only on text, not elements! *) + let body = + let subst text = + "" ^ text ^ "" + in + List.map (fun str -> + if String.length str > 0 && str.[0] != '<' then + Pcre.substitute ~rex ~subst str + else + str) body in + + (* Join the XHTML fragments back together again. *) + String.concat "" (List.concat [ head ; body ; tail ]) + in + + (* Check the templates table for extensions. *) + let get_extension url = + try + let name = + List.hd ( + PGSQL(dbh) "select extension from templates + where $url ~ url_regexp + order by ordering + limit 1" + ) in + Some (List.assoc name !extensions) + with + Not_found | ExtList.List.Empty_list | Failure "hd" -> None + in (* This code generates ordinary pages. *) let make_page title description pageid last_modified_date has_page_css - version page page' = + version page page' extension = let t = template_page in + let th = template_page_header in t#set "title" title; - t#set "description" description; - t#set "pageid" (string_of_int pageid); - t#set "last_modified_date" (printable_date last_modified_date); + th#set "title" title; + t#set "last_modified_date" last_modified_date; + + (match description with + None -> th#conditional "has_description" false + | Some description -> + th#conditional "has_description" true; + th#set "description" description); if page <> page' then (* redirection *) ( t#set "page" page'; + th#set "page" page'; t#set "original_page" page; (* XXX title - get it from database *) t#conditional "redirected" true ) else ( t#set "page" page; + th#set "page" page; t#conditional "redirected" false ); - t#conditional "has_host_css" has_host_css; - t#conditional "has_page_css" has_page_css; + th#conditional "has_page_css" has_page_css; + + (* Are we showing an old version of the page? If so, warn. *) + (match version with + None -> + t#conditional "is_old_version" false; + th#conditional "is_old_version" false + | Some pageid -> + t#conditional "is_old_version" true; + th#conditional "is_old_version" true; + t#set "old_version" (Int32.to_string pageid); + th#set "old_version" (Int32.to_string pageid)); + + (* Just before we show the header, call any registered pre-page + * handlers. They might want to send cookies. + *) + List.iter (fun handler -> + handler r q dbh hostid page') !pre_page_handlers; + + (* At this point, we can print out the header and flush it back to + * the user, allowing the browser to start fetching stylesheets + * and background images while we compose the page. + *) + q#header (); + ignore (print_string r th#to_string); + ignore (Request.rflush r); + + t#conditional "has_feedback_email" has_feedback_email; + t#conditional "mailing_list" mailing_list; + t#conditional "navigation" navigation; t#conditional "can_edit" can_edit; t#conditional "can_manage_users" can_manage_users; + t#conditional "has_stats" has_stats; (* Pull out the sections in this page. *) - let sth = dbh#prepare_cached - "select ordering, sectionname, content, divname - from contents - where pageid = ? - order by ordering" in - sth#execute [`Int pageid]; + let sections = + match pageid with + None -> [] + | Some pageid -> + let rows = PGSQL(dbh) + "select ordering, sectionname, content, divname, jsgo + from contents where pageid = $pageid order by ordering" in + + List.map + (fun (ordering, sectionname, content, divname, jsgo) -> + let divname, has_divname = + match divname with + | None -> "", false + | Some divname -> divname, true in + let jsgo, has_jsgo = + match jsgo with + | None -> "", false + | Some jsgo -> jsgo, true in + let sectionname, has_sectionname = + match sectionname with + | None -> "", false + | Some sectionname -> sectionname, true in + let linkname = linkname_of_sectionname sectionname in + [ "ordering", Template.VarString (Int32.to_string ordering); + "has_sectionname", + Template.VarConditional has_sectionname; + "sectionname", Template.VarString sectionname; + "linkname", Template.VarString linkname; + "content", + Template.VarString + (Wikilib.xhtml_of_content r dbh hostid content); + "has_divname", Template.VarConditional has_divname; + "divname", Template.VarString divname; + "has_jsgo", Template.VarConditional has_jsgo; + "jsgo", Template.VarString jsgo ]) rows in + (* Call an extension to generate the first section in this page? *) let sections = - sth#map - (function [`Int ordering; `String sectionname; `String content; - (`Null | `String _) as divname] -> - let divname, has_divname = - match divname with - `Null -> "", false - | `String divname -> divname, true in - [ "ordering", Template.VarString (string_of_int ordering); - "sectionname", Template.VarString sectionname; - "content", - Template.VarString - (Wikilib.xhtml_of_content dbh hostid content); - "has_divname", Template.VarConditional has_divname; - "divname", Template.VarString divname ] - | _ -> assert false) in + match extension with + None -> sections + | Some extension -> + let content = extension r dbh hostid page' in + let section = [ + "ordering", Template.VarString "0"; + "has_sectionname", Template.VarConditional false; + "linkname", Template.VarString ""; + "content", Template.VarString content; + "has_divname", Template.VarConditional true; + "divname", Template.VarString "form_div"; + "has_jsgo", Template.VarConditional false; + "jsgo", Template.VarString ""; + ] in + section :: sections in t#table "sections" sections; - (* Are we showing an old version of the page? If so, warn. *) - (match version with - None -> - t#conditional "is_old_version" false - | Some pageid -> - t#conditional "is_old_version" true; - t#set "old_version" (string_of_int pageid)); - (* Login status. *) (match user with Anonymous -> t#conditional "user_logged_in" false - | User (_, username, _) -> + | User (_, username, _, _) -> t#conditional "user_logged_in" true; t#set "username" username); - q#template t + (* Can anonymous users create accounts? If not them we don't + * want to offer to create accounts for them. + *) + t#conditional "create_account_anon" host.create_account_anon; + + (* If logged in, we want to update the recently_visited table. *) + if pageid <> None then ( + match user with + | User (userid, _, _, _) -> + (try + PGSQL(dbh) + "delete from recently_visited + where hostid = $hostid and userid = $userid + and url = $page'"; + PGSQL(dbh) + "insert into recently_visited (hostid, userid, url) + values ($hostid, $userid, $page')"; + PGOCaml.commit dbh; + with + exn -> + (* Exceptions here are non-fatal. Just print them. *) + prerr_endline "exception updating recently_visited:"; + prerr_endline (Printexc.to_string exn); + PGOCaml.rollback dbh; + ); + PGOCaml.begin_work dbh; + | _ -> () + ); + + (* Navigation links. *) + if navigation then ( + let max_links = 18 in (* Show no more links than this. *) + + (* What links here. *) + let wlh = what_links_here dbh hostid page' in + let wlh = List.take max_links wlh in + let wlh_urls = List.map fst wlh in (* Just the URLs ... *) + + let rv = + match user with + | User (userid, _, _, _) -> + (* Recently visited URLs, but don't repeat any from the 'what + * links here' section, and don't link to self. + *) + let not_urls = page' :: wlh_urls in + let limit = Int32.of_int (max_links - List.length wlh_urls) in + let rows = + PGSQL(dbh) + "select rv.url, p.title, rv.visit_time + from recently_visited rv, pages p + where rv.hostid = $hostid and rv.userid = $userid + and rv.url not in $@not_urls + and rv.hostid = p.hostid and rv.url = p.url + order by 3 desc + limit $limit" in + List.map ( + fun (url, title, _) -> url, title + ) rows + | _ -> [] in + + (* Links to page. *) + let f (page, title) = [ "page", Template.VarString page; + "title", Template.VarString title ] in + let table = List.map f wlh in + t#table "what_links_here" table; + t#conditional "has_what_links_here" (wlh <> []); + + let table = List.map f rv in + t#table "recently_visited" table; + t#conditional "has_recently_visited" (rv <> []); + + (* If both lists are empty (ie. an empty navigation box would + * appear), then disable navigation altogether. + *) + if wlh = [] && rv = [] then t#conditional "navigation" false + ); + + (* If we are coming from a search engine then we want to highlight + * search terms throughout the whole page ... + *) + try + let referer = Table.get (Request.headers_in r) "Referer" in + let search_terms = search_terms_from_referer referer in + + (* Highlight the search terms. *) + let xhtml = t#to_string in + let xhtml = highlight_search_terms xhtml search_terms "search_term" in + + (* Deliver the page. *) + ignore (print_string r xhtml) + with + Not_found -> + (* No referer / no search terms / not a search engine referer. *) + ignore (print_string r t#to_string) in (* This code generates 404 pages. *) let make_404 () = Request.set_status r 404; (* Return a 404 error code. *) - let t = template_404 in - t#set "page" page; + let th = template_404_header in + th#set "page" page; let search_terms = String.map @@ -126,81 +417,187 @@ let run r (q : cgi) (dbh : Dbi.connection) hostid {edit_anon=edit_anon} user = ('a'..'z' | 'A'..'Z' | '0'..'9') as c -> c | _ -> ' ') page in - t#set "search_terms" search_terms; + th#set "search_terms" search_terms; - t#conditional "has_host_css" has_host_css; + (* Flush out the header while we start the search. *) + q#header (); + ignore (print_string r th#to_string); + ignore (Request.rflush r); - t#conditional "can_edit" can_edit; - t#conditional "can_manage_users" can_manage_users; + let t = template_404 in + t#set "query" search_terms; + t#set "canonical_hostname" host.canonical_hostname; - q#template t + (* This is a simplified version of the code in search.ml. *) + let have_results = + (* Get the keywords from the query string. *) + let keywords = Pcre.split ~rex:split_words search_terms in + let keywords = + List.filter (fun s -> not (string_is_whitespace s)) keywords in + let keywords = List.map String.lowercase keywords in + + (* Turn the keywords into a tsearch2 ts_query string. *) + let tsquery = String.concat "&" keywords in + + (* Search the titles first. *) + let rows = + PGSQL(dbh) + "select url, title, last_modified_date, + (lower (title) = lower ($search_terms)) as exact + from pages + where hostid = $hostid + and url is not null + and redirect is null + and title_description_fti @@ to_tsquery ('default', $tsquery) + order by exact desc, last_modified_date desc, title" in + + let titles = + List.map (function + | (Some url, title, last_modified, _) -> + url, title, last_modified + | _ -> assert false) rows in + + let have_titles = titles <> [] in + t#conditional "have_titles" have_titles; + + (* Search the contents. *) + let rows = + PGSQL(dbh) + "select c.id, p.url, p.title, p.last_modified_date + from contents c, pages p + where c.pageid = p.id + and p.hostid = $hostid + and url is not null + and p.redirect is null + and c.content_fti @@ to_tsquery ('default', $tsquery) + order by p.last_modified_date desc, p.title + limit 50" in + + let contents = + List.map (function + | (contentid, Some url, title, last_modified) -> + contentid, url, title, last_modified + | _ -> assert false) rows in + + let have_contents = contents <> [] in + t#conditional "have_contents" have_contents; + + (* Pull out the actual text which matched so we can generate a summary. + * XXX tsearch2 can actually do better than this by emboldening + * the text which maps. + *) + let content_map = + if contents = [] then [] + else ( + let rows = + let contentids = + List.map (fun (contentid, _,_,_) -> contentid) contents in + PGSQL(dbh) + "select id, sectionname, content from contents + where id in $@contentids" in + List.map (fun (id, sectionname, content) -> + id, (sectionname, content)) rows + ) in + + (* Generate the final tables. *) + let table = + List.map (fun (url, title, last_modified) -> + let last_modified = printable_date last_modified in + [ "url", Template.VarString url; + "title", Template.VarString title; + "last_modified", Template.VarString last_modified ] + ) titles in + t#table "titles" table; + + let table = + List.map + (fun (contentid, url, title, last_modified) -> + let sectionname, content = List.assoc contentid content_map in + let have_sectionname, sectionname = + match sectionname with + None -> false, "" + | Some sectionname -> true, sectionname in + let content = + truncate 160 + (Wikilib.text_of_xhtml + (Wikilib.xhtml_of_content r dbh hostid content)) in + let linkname = linkname_of_sectionname sectionname in + let last_modified = printable_date last_modified in + [ "url", Template.VarString url; + "title", Template.VarString title; + "have_sectionname", Template.VarConditional have_sectionname; + "sectionname", Template.VarString sectionname; + "linkname", Template.VarString linkname; + "content", Template.VarString content; + "last_modified", Template.VarString last_modified ] + ) contents in + t#table "contents" table; + + (* Do we have any results? *) + let have_results = have_titles || have_contents in + have_results in + t#conditional "have_results" have_results; + + (* Deliver the rest of the page. *) + ignore (print_string r t#to_string) in (* Fetch a page by name. This function can give three answers: * (1) Page fetched OK (fetches some details of the page). * (2) Page is a redirect (fetches the name of the redirect page). - * (3) Page not found in database, ie. 404 error. - *) - (* XXX Should do a case-insensitive matching of URLs, and if the URL differs - * in case only should redirect to the lowercase version. + * (3) Page not found in database, could be template or 404 error. *) let fetch_page page version allow_redirect = match version with | None -> if allow_redirect then ( - let sth = - dbh#prepare_cached - "select redirect, id, title, description, last_modified_date, - css is not null - from pages where hostid = ? and url = ?" in - sth#execute [`Int hostid; `String page]; - (try - (match sth#fetch1 () with - | [ `Null; `Int id; `String title; `String description; - `Timestamp last_modified_date; `Bool has_page_css ] -> - FPOK (id, title, description, last_modified_date, - has_page_css) - | `String redirect :: _ -> - FPRedirect redirect - | _ -> assert false) - with - Not_found -> FPNotFound) + let rows = PGSQL(dbh) + "select url, redirect, id, title, description, + last_modified_date, css is not null + from pages + where hostid = $hostid and lower (url) = lower ($page)" in + match rows with + | [Some page', _, _, _, _, _, _] + when page <> page' -> (* different case *) + FPExternalRedirect page' + | [ _, None, id, title, description, + last_modified_date, has_page_css ] -> + let has_page_css = Option.get has_page_css in + FPOK (id, title, description, last_modified_date, + has_page_css) + | [_, Some redirect, _, _, _, _, _] -> + FPInternalRedirect redirect + | [] -> FPNotFound + | _ -> assert false ) else (* redirects not allowed ... *) ( - let sth = - dbh#prepare_cached - "select id, title, description, last_modified_date, - css is not null - from pages where hostid = ? and url = ?" in - sth#execute [`Int hostid; `String page]; - (try - (match sth#fetch1 () with - | [ `Int id; `String title; `String description; - `Timestamp last_modified_date; `Bool has_page_css ] -> - FPOK (id, title, description, last_modified_date, - has_page_css) - | _ -> assert false) - with - Not_found -> FPNotFound) + let rows = PGSQL(dbh) + "select id, title, description, last_modified_date, + css is not null + from pages where hostid = $hostid and url = $page" in + match rows with + | [ id, title, description, + last_modified_date, has_page_css ] -> + let has_page_css = Option.get has_page_css in + FPOK (id, title, description, last_modified_date, + has_page_css) + | [] -> FPNotFound + | _ -> assert false ) | Some version -> - let sth = - dbh#prepare_cached - "select id, title, description, last_modified_date, - css is not null - from pages - where hostid = ? and id = ? and - (url = ? or url_deleted = ?)" in - sth#execute [`Int hostid; `Int version; - `String page; `String page]; - (try - (match sth#fetch1 () with - | [ `Int id; `String title; `String description; - `Timestamp last_modified_date; `Bool has_page_css ] -> - FPOK (id, title, description, last_modified_date, - has_page_css) - | _ -> assert false) - with - Not_found -> FPNotFound) + let rows = PGSQL(dbh) + "select id, title, description, last_modified_date, + css is not null + from pages + where hostid = $hostid and id = $version and + (url = $page or url_deleted = $page)" in + match rows with + | [ id, title, description, + last_modified_date, has_page_css ] -> + let has_page_css = Option.get has_page_css in + FPOK (id, title, description, last_modified_date, + has_page_css) + | [] -> FPNotFound + | _ -> assert false in (* Here we deal with the complex business of redirects and versions. *) @@ -208,27 +605,45 @@ let run r (q : cgi) (dbh : Dbi.connection) hostid {edit_anon=edit_anon} user = let allow_redirect, version = if can_edit then ( not (q#param_true "no_redirect"), - try Some (int_of_string (q#param "version")) with Not_found -> None + try Some (Int32.of_string (q#param "version")) with Not_found -> None ) else (true, None) in let rec loop page' i = if i > max_redirect then ( error ~title:"Too many redirections" ~back_button:true - q ("Too many redirects between pages. This may happen because " ^ - "of a cycle of redirections."); - raise CgiExit + dbh hostid q + ("Too many redirects between pages. This may happen because " ^ + "of a cycle of redirections."); + return () ) else match fetch_page page' version allow_redirect with | FPOK (pageid, title, description, last_modified_date, has_page_css)-> - make_page title description pageid last_modified_date has_page_css - version page page' - | FPRedirect page' -> + (* Check if the page is also a template. *) + let extension = get_extension page' in + make_page title (Some description) (Some pageid) + (printable_date last_modified_date) has_page_css + version page page' extension + | FPInternalRedirect page' -> loop page' (i+1) + | FPExternalRedirect page' -> + (* This normally happens when a user has requested an uppercase + * page name. We redirect to the true (lowercase) version. + *) + q#redirect ("http://" ^ host.hostname ^ "/" ^ page') | FPNotFound -> - make_404 () + (* Might be a templated page with no content in it. *) + let extension = get_extension page' in + (match extension with + | (Some _) as extension -> + let title = page' in + make_page title None None + "Now" false None page page' + extension + | None -> + make_404 ()) in loop page 0 let () = - register_script run + register_script ~restrict:[CanView] run