- (* Only interested in the tags with class="internal". *)
- let html =
- List.filter (fun str ->
- Pcre.pmatch ~rex:internal_re str
- && Pcre.pmatch ~rex:href_re str)
- html in
-
- (* Extract the URL names. *)
- let links = List.map (fun str ->
- let subs =
- try Pcre.exec ~rex:href_re str
- with Not_found -> assert false in
- Pcre.get_substring subs 1) html in
-
- (* Return the list of links. *)
- links
+ (* Only interested in the tags with class="internal" or class="newpage". *)
+ let internal_links =
+ let html =
+ List.filter (fun str ->
+ Pcre.pmatch ~rex:internal_re str
+ && Pcre.pmatch ~rex:href_re str)
+ html in
+
+ (* Extract the URL names. *)
+ List.map (fun str ->
+ let subs =
+ try Pcre.exec ~rex:href_re str
+ with Not_found -> assert false in
+ Pcre.get_substring subs 1) html in
+
+ let newpage_links =
+ let html =
+ List.filter (fun str ->
+ Pcre.pmatch ~rex:newpage_re str
+ && Pcre.pmatch ~rex:title_re str)
+ html in
+
+ (* Extract the titles. *)
+ let titles =
+ List.map (fun str ->
+ let subs =
+ try Pcre.exec ~rex:title_re str
+ with Not_found -> assert false in
+ Pcre.get_substring subs 1) html in
+
+ (* Map the titles to URLs. *)
+ List.filter_map
+ (fun title ->
+ match Wikilib.generate_url_of_title dbh hostid title with
+ | Wikilib.GenURL_OK url -> Some url
+ | _ -> None) titles in
+
+ (* Return the complete list of links. *)
+ internal_links @ newpage_links