From 97b7d5c0380ac0fdc846306689d3d8015b6b6e62 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 5 Mar 2014 16:48:41 +0400 Subject: split functions php into two parts because syntastic is too slow get_feeds_from_html: stricter feed detection based on whatwg guidelines --- include/functions2.php | 2397 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2397 insertions(+) create mode 100644 include/functions2.php (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php new file mode 100644 index 000000000..4e9796147 --- /dev/null +++ b/include/functions2.php @@ -0,0 +1,2397 @@ + array( + "next_feed" => __("Open next feed"), + "prev_feed" => __("Open previous feed"), + "next_article" => __("Open next article"), + "prev_article" => __("Open previous article"), + "next_article_noscroll" => __("Open next article (don't scroll long articles)"), + "prev_article_noscroll" => __("Open previous article (don't scroll long articles)"), + "next_article_noexpand" => __("Move to next article (don't expand or mark read)"), + "prev_article_noexpand" => __("Move to previous article (don't expand or mark read)"), + "search_dialog" => __("Show search dialog")), + __("Article") => array( + "toggle_mark" => __("Toggle starred"), + "toggle_publ" => __("Toggle published"), + "toggle_unread" => __("Toggle unread"), + "edit_tags" => __("Edit tags"), + "dismiss_selected" => __("Dismiss selected"), + "dismiss_read" => __("Dismiss read"), + "open_in_new_window" => __("Open in new window"), + "catchup_below" => __("Mark below as read"), + "catchup_above" => __("Mark above as read"), + "article_scroll_down" => __("Scroll down"), + "article_scroll_up" => __("Scroll up"), + "select_article_cursor" => __("Select article under cursor"), + "email_article" => __("Email article"), + "close_article" => __("Close/collapse article"), + "toggle_expand" => __("Toggle article expansion (combined mode)"), + "toggle_widescreen" => __("Toggle widescreen mode"), + "toggle_embed_original" => __("Toggle embed original")), + __("Article selection") => array( + "select_all" => __("Select all articles"), + "select_unread" => __("Select unread"), + "select_marked" => __("Select starred"), + "select_published" => __("Select published"), + "select_invert" => __("Invert selection"), + "select_none" => __("Deselect everything")), + __("Feed") => array( + "feed_refresh" => __("Refresh current feed"), + "feed_unhide_read" => __("Un/hide read feeds"), + "feed_subscribe" => __("Subscribe to feed"), + "feed_edit" => __("Edit feed"), + "feed_catchup" => __("Mark as read"), + "feed_reverse" => __("Reverse headlines"), + "feed_debug_update" => __("Debug feed update"), + "catchup_all" => __("Mark all feeds as read"), + "cat_toggle_collapse" => __("Un/collapse current category"), + "toggle_combined_mode" => __("Toggle combined mode"), + "toggle_cdm_expanded" => __("Toggle auto expand in combined mode")), + __("Go to") => array( + "goto_all" => __("All articles"), + "goto_fresh" => __("Fresh"), + "goto_marked" => __("Starred"), + "goto_published" => __("Published"), + "goto_tagcloud" => __("Tag cloud"), + "goto_prefs" => __("Preferences")), + __("Other") => array( + "create_label" => __("Create label"), + "create_filter" => __("Create filter"), + "collapse_sidebar" => __("Un/collapse sidebar"), + "help_dialog" => __("Show help dialog")) + ); + + foreach (PluginHost::getInstance()->get_hooks(PluginHost::HOOK_HOTKEY_INFO) as $plugin) { + $hotkeys = $plugin->hook_hotkey_info($hotkeys); + } + + return $hotkeys; + } + + function get_hotkeys_map() { + $hotkeys = array( +// "navigation" => array( + "k" => "next_feed", + "j" => "prev_feed", + "n" => "next_article", + "p" => "prev_article", + "(38)|up" => "prev_article", + "(40)|down" => "next_article", +// "^(38)|Ctrl-up" => "prev_article_noscroll", +// "^(40)|Ctrl-down" => "next_article_noscroll", + "(191)|/" => "search_dialog", +// "article" => array( + "s" => "toggle_mark", + "*s" => "toggle_publ", + "u" => "toggle_unread", + "*t" => "edit_tags", + "*d" => "dismiss_selected", + "*x" => "dismiss_read", + "o" => "open_in_new_window", + "c p" => "catchup_below", + "c n" => "catchup_above", + "*n" => "article_scroll_down", + "*p" => "article_scroll_up", + "*(38)|Shift+up" => "article_scroll_up", + "*(40)|Shift+down" => "article_scroll_down", + "a *w" => "toggle_widescreen", + "a e" => "toggle_embed_original", + "e" => "email_article", + "a q" => "close_article", +// "article_selection" => array( + "a a" => "select_all", + "a u" => "select_unread", + "a *u" => "select_marked", + "a p" => "select_published", + "a i" => "select_invert", + "a n" => "select_none", +// "feed" => array( + "f r" => "feed_refresh", + "f a" => "feed_unhide_read", + "f s" => "feed_subscribe", + "f e" => "feed_edit", + "f q" => "feed_catchup", + "f x" => "feed_reverse", + "f *d" => "feed_debug_update", + "f *c" => "toggle_combined_mode", + "f c" => "toggle_cdm_expanded", + "*q" => "catchup_all", + "x" => "cat_toggle_collapse", +// "goto" => array( + "g a" => "goto_all", + "g f" => "goto_fresh", + "g s" => "goto_marked", + "g p" => "goto_published", + "g t" => "goto_tagcloud", + "g *p" => "goto_prefs", +// "other" => array( + "(9)|Tab" => "select_article_cursor", // tab + "c l" => "create_label", + "c f" => "create_filter", + "c s" => "collapse_sidebar", + "^(191)|Ctrl+/" => "help_dialog", + ); + + if (get_pref('COMBINED_DISPLAY_MODE')) { + $hotkeys["^(38)|Ctrl-up"] = "prev_article_noscroll"; + $hotkeys["^(40)|Ctrl-down"] = "next_article_noscroll"; + } + + foreach (PluginHost::getInstance()->get_hooks(PluginHost::HOOK_HOTKEY_MAP) as $plugin) { + $hotkeys = $plugin->hook_hotkey_map($hotkeys); + } + + $prefixes = array(); + + foreach (array_keys($hotkeys) as $hotkey) { + $pair = explode(" ", $hotkey, 2); + + if (count($pair) > 1 && !in_array($pair[0], $prefixes)) { + array_push($prefixes, $pair[0]); + } + } + + return array($prefixes, $hotkeys); + } + + function make_runtime_info() { + $data = array(); + + $result = db_query("SELECT MAX(id) AS mid, COUNT(*) AS nf FROM + ttrss_feeds WHERE owner_uid = " . $_SESSION["uid"]); + + $max_feed_id = db_fetch_result($result, 0, "mid"); + $num_feeds = db_fetch_result($result, 0, "nf"); + + $data["max_feed_id"] = (int) $max_feed_id; + $data["num_feeds"] = (int) $num_feeds; + + $data['last_article_id'] = getLastArticleId(); + $data['cdm_expanded'] = get_pref('CDM_EXPANDED'); + + $data['dep_ts'] = calculate_dep_timestamp(); + $data['reload_on_ts_change'] = !defined('_NO_RELOAD_ON_TS_CHANGE'); + + if (file_exists(LOCK_DIRECTORY . "/update_daemon.lock")) { + + $data['daemon_is_running'] = (int) file_is_locked("update_daemon.lock"); + + if (time() - $_SESSION["daemon_stamp_check"] > 30) { + + $stamp = (int) @file_get_contents(LOCK_DIRECTORY . "/update_daemon.stamp"); + + if ($stamp) { + $stamp_delta = time() - $stamp; + + if ($stamp_delta > 1800) { + $stamp_check = 0; + } else { + $stamp_check = 1; + $_SESSION["daemon_stamp_check"] = time(); + } + + $data['daemon_stamp_ok'] = $stamp_check; + + $stamp_fmt = date("Y.m.d, G:i", $stamp); + + $data['daemon_stamp'] = $stamp_fmt; + } + } + } + + if ($_SESSION["last_version_check"] + 86400 + rand(-1000, 1000) < time()) { + $new_version_details = @check_for_update(); + + $data['new_version_available'] = (int) ($new_version_details != false); + + $_SESSION["last_version_check"] = time(); + $_SESSION["version_data"] = $new_version_details; + } + + return $data; + } + + function search_to_sql($search) { + + $search_query_part = ""; + + $keywords = str_getcsv($search, " "); + $query_keywords = array(); + $search_words = array(); + + foreach ($keywords as $k) { + if (strpos($k, "-") === 0) { + $k = substr($k, 1); + $not = "NOT"; + } else { + $not = ""; + } + + $commandpair = explode(":", mb_strtolower($k), 2); + + switch ($commandpair[0]) { + case "title": + if ($commandpair[1]) { + array_push($query_keywords, "($not (LOWER(ttrss_entries.title) LIKE '%". + db_escape_string(mb_strtolower($commandpair[1]))."%'))"); + } else { + array_push($query_keywords, "(UPPER(ttrss_entries.title) $not LIKE UPPER('%$k%') + OR UPPER(ttrss_entries.content) $not LIKE UPPER('%$k%'))"); + array_push($search_words, $k); + } + break; + case "author": + if ($commandpair[1]) { + array_push($query_keywords, "($not (LOWER(author) LIKE '%". + db_escape_string(mb_strtolower($commandpair[1]))."%'))"); + } else { + array_push($query_keywords, "(UPPER(ttrss_entries.title) $not LIKE UPPER('%$k%') + OR UPPER(ttrss_entries.content) $not LIKE UPPER('%$k%'))"); + array_push($search_words, $k); + } + break; + case "note": + if ($commandpair[1]) { + if ($commandpair[1] == "true") + array_push($query_keywords, "($not (note IS NOT NULL AND note != ''))"); + else if ($commandpair[1] == "false") + array_push($query_keywords, "($not (note IS NULL OR note = ''))"); + else + array_push($query_keywords, "($not (LOWER(note) LIKE '%". + db_escape_string(mb_strtolower($commandpair[1]))."%'))"); + } else { + array_push($query_keywords, "(UPPER(ttrss_entries.title) $not LIKE UPPER('%$k%') + OR UPPER(ttrss_entries.content) $not LIKE UPPER('%$k%'))"); + if (!$not) array_push($search_words, $k); + } + break; + case "star": + + if ($commandpair[1]) { + if ($commandpair[1] == "true") + array_push($query_keywords, "($not (marked = true))"); + else + array_push($query_keywords, "($not (marked = false))"); + } else { + array_push($query_keywords, "(UPPER(ttrss_entries.title) $not LIKE UPPER('%$k%') + OR UPPER(ttrss_entries.content) $not LIKE UPPER('%$k%'))"); + if (!$not) array_push($search_words, $k); + } + break; + case "pub": + if ($commandpair[1]) { + if ($commandpair[1] == "true") + array_push($query_keywords, "($not (published = true))"); + else + array_push($query_keywords, "($not (published = false))"); + + } else { + array_push($query_keywords, "(UPPER(ttrss_entries.title) $not LIKE UPPER('%$k%') + OR UPPER(ttrss_entries.content) $not LIKE UPPER('%$k%'))"); + if (!$not) array_push($search_words, $k); + } + break; + default: + if (strpos($k, "@") === 0) { + + $user_tz_string = get_pref('USER_TIMEZONE', $_SESSION['uid']); + $orig_ts = strtotime(substr($k, 1)); + $k = date("Y-m-d", convert_timestamp($orig_ts, $user_tz_string, 'UTC')); + + //$k = date("Y-m-d", strtotime(substr($k, 1))); + + array_push($query_keywords, "(".SUBSTRING_FOR_DATE."(updated,1,LENGTH('$k')) $not = '$k')"); + } else { + array_push($query_keywords, "(UPPER(ttrss_entries.title) $not LIKE UPPER('%$k%') + OR UPPER(ttrss_entries.content) $not LIKE UPPER('%$k%'))"); + + if (!$not) array_push($search_words, $k); + } + } + } + + $search_query_part = implode("AND", $query_keywords); + + return array($search_query_part, $search_words); + } + + function getParentCategories($cat, $owner_uid) { + $rv = array(); + + $result = db_query("SELECT parent_cat FROM ttrss_feed_categories + WHERE id = '$cat' AND parent_cat IS NOT NULL AND owner_uid = $owner_uid"); + + while ($line = db_fetch_assoc($result)) { + array_push($rv, $line["parent_cat"]); + $rv = array_merge($rv, getParentCategories($line["parent_cat"], $owner_uid)); + } + + return $rv; + } + + function getChildCategories($cat, $owner_uid) { + $rv = array(); + + $result = db_query("SELECT id FROM ttrss_feed_categories + WHERE parent_cat = '$cat' AND owner_uid = $owner_uid"); + + while ($line = db_fetch_assoc($result)) { + array_push($rv, $line["id"]); + $rv = array_merge($rv, getChildCategories($line["id"], $owner_uid)); + } + + return $rv; + } + + function queryFeedHeadlines($feed, $limit, $view_mode, $cat_view, $search, $search_mode, $override_order = false, $offset = 0, $owner_uid = 0, $filter = false, $since_id = 0, $include_children = false, $ignore_vfeed_group = false, $override_strategy = false, $override_vfeed = false) { + + if (!$owner_uid) $owner_uid = $_SESSION["uid"]; + + $ext_tables_part = ""; + $search_words = array(); + + if ($search) { + + if (SPHINX_ENABLED) { + $ids = join(",", @sphinx_search($search, 0, 500)); + + if ($ids) + $search_query_part = "ref_id IN ($ids) AND "; + else + $search_query_part = "ref_id = -1 AND "; + + } else { + list($search_query_part, $search_words) = search_to_sql($search); + $search_query_part .= " AND "; + } + + } else { + $search_query_part = ""; + } + + if ($filter) { + + if (DB_TYPE == "pgsql") { + $query_strategy_part .= " AND updated > NOW() - INTERVAL '14 days' "; + } else { + $query_strategy_part .= " AND updated > DATE_SUB(NOW(), INTERVAL 14 DAY) "; + } + + $override_order = "updated DESC"; + + $filter_query_part = filter_to_sql($filter, $owner_uid); + + // Try to check if SQL regexp implementation chokes on a valid regexp + + + $result = db_query("SELECT true AS true_val FROM ttrss_entries, + ttrss_user_entries, ttrss_feeds + WHERE $filter_query_part LIMIT 1", false); + + if ($result) { + $test = db_fetch_result($result, 0, "true_val"); + + if (!$test) { + $filter_query_part = "false AND"; + } else { + $filter_query_part .= " AND"; + } + } else { + $filter_query_part = "false AND"; + } + + } else { + $filter_query_part = ""; + } + + if ($since_id) { + $since_id_part = "ttrss_entries.id > $since_id AND "; + } else { + $since_id_part = ""; + } + + $view_query_part = ""; + + if ($view_mode == "adaptive") { + if ($search) { + $view_query_part = " "; + } else if ($feed != -1) { + + $unread = getFeedUnread($feed, $cat_view); + + if ($cat_view && $feed > 0 && $include_children) + $unread += getCategoryChildrenUnread($feed); + + if ($unread > 0) + $view_query_part = " unread = true AND "; + + } + } + + if ($view_mode == "marked") { + $view_query_part = " marked = true AND "; + } + + if ($view_mode == "has_note") { + $view_query_part = " (note IS NOT NULL AND note != '') AND "; + } + + if ($view_mode == "published") { + $view_query_part = " published = true AND "; + } + + if ($view_mode == "unread" && $feed != -6) { + $view_query_part = " unread = true AND "; + } + + if ($limit > 0) { + $limit_query_part = "LIMIT " . $limit; + } + + $allow_archived = false; + + $vfeed_query_part = ""; + + // override query strategy and enable feed display when searching globally + if ($search && $search_mode == "all_feeds") { + $query_strategy_part = "true"; + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + /* tags */ + } else if (!is_numeric($feed)) { + $query_strategy_part = "true"; + $vfeed_query_part = "(SELECT title FROM ttrss_feeds WHERE + id = feed_id) as feed_title,"; + } else if ($search && $search_mode == "this_cat") { + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + + if ($feed > 0) { + if ($include_children) { + $subcats = getChildCategories($feed, $owner_uid); + array_push($subcats, $feed); + $cats_qpart = join(",", $subcats); + } else { + $cats_qpart = $feed; + } + + $query_strategy_part = "ttrss_feeds.cat_id IN ($cats_qpart)"; + + } else { + $query_strategy_part = "ttrss_feeds.cat_id IS NULL"; + } + + } else if ($feed > 0) { + + if ($cat_view) { + + if ($feed > 0) { + if ($include_children) { + # sub-cats + $subcats = getChildCategories($feed, $owner_uid); + + array_push($subcats, $feed); + $query_strategy_part = "cat_id IN (". + implode(",", $subcats).")"; + + } else { + $query_strategy_part = "cat_id = '$feed'"; + } + + } else { + $query_strategy_part = "cat_id IS NULL"; + } + + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + + } else { + $query_strategy_part = "feed_id = '$feed'"; + } + } else if ($feed == 0 && !$cat_view) { // archive virtual feed + $query_strategy_part = "feed_id IS NULL"; + $allow_archived = true; + } else if ($feed == 0 && $cat_view) { // uncategorized + $query_strategy_part = "cat_id IS NULL AND feed_id IS NOT NULL"; + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + } else if ($feed == -1) { // starred virtual feed + $query_strategy_part = "marked = true"; + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + $allow_archived = true; + + if (!$override_order) { + $override_order = "last_marked DESC, date_entered DESC, updated DESC"; + } + + } else if ($feed == -2) { // published virtual feed OR labels category + + if (!$cat_view) { + $query_strategy_part = "published = true"; + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + $allow_archived = true; + + if (!$override_order) { + $override_order = "last_published DESC, date_entered DESC, updated DESC"; + } + + } else { + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + + $ext_tables_part = ",ttrss_labels2,ttrss_user_labels2"; + + $query_strategy_part = "ttrss_labels2.id = ttrss_user_labels2.label_id AND + ttrss_user_labels2.article_id = ref_id"; + + } + } else if ($feed == -6) { // recently read + $query_strategy_part = "unread = false AND last_read IS NOT NULL"; + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + $allow_archived = true; + + if (!$override_order) $override_order = "last_read DESC"; + +/* } else if ($feed == -7) { // shared + $query_strategy_part = "uuid != ''"; + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + $allow_archived = true; */ + } else if ($feed == -3) { // fresh virtual feed + $query_strategy_part = "unread = true AND score >= 0"; + + $intl = get_pref("FRESH_ARTICLE_MAX_AGE", $owner_uid); + + if (DB_TYPE == "pgsql") { + $query_strategy_part .= " AND date_entered > NOW() - INTERVAL '$intl hour' "; + } else { + $query_strategy_part .= " AND date_entered > DATE_SUB(NOW(), INTERVAL $intl HOUR) "; + } + + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + } else if ($feed == -4) { // all articles virtual feed + $allow_archived = true; + $query_strategy_part = "true"; + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + } else if ($feed <= LABEL_BASE_INDEX) { // labels + $label_id = feed_to_label_id($feed); + + $query_strategy_part = "label_id = '$label_id' AND + ttrss_labels2.id = ttrss_user_labels2.label_id AND + ttrss_user_labels2.article_id = ref_id"; + + $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; + $ext_tables_part = ",ttrss_labels2,ttrss_user_labels2"; + $allow_archived = true; + + } else { + $query_strategy_part = "true"; + } + + $order_by = "score DESC, date_entered DESC, updated DESC"; + + if ($view_mode == "unread_first") { + $order_by = "unread DESC, $order_by"; + } + + if ($override_order) { + $order_by = $override_order; + } + + if ($override_strategy) { + $query_strategy_part = $override_strategy; + } + + if ($override_vfeed) { + $vfeed_query_part = $override_vfeed; + } + + $feed_title = ""; + + if ($search) { + $feed_title = T_sprintf("Search results: %s", $search); + } else { + if ($cat_view) { + $feed_title = getCategoryTitle($feed); + } else { + if (is_numeric($feed) && $feed > 0) { + $result = db_query("SELECT title,site_url,last_error,last_updated + FROM ttrss_feeds WHERE id = '$feed' AND owner_uid = $owner_uid"); + + $feed_title = db_fetch_result($result, 0, "title"); + $feed_site_url = db_fetch_result($result, 0, "site_url"); + $last_error = db_fetch_result($result, 0, "last_error"); + $last_updated = db_fetch_result($result, 0, "last_updated"); + } else { + $feed_title = getFeedTitle($feed); + } + } + } + + + $content_query_part = "content, "; + + + if (is_numeric($feed)) { + + if ($feed >= 0) { + $feed_kind = "Feeds"; + } else { + $feed_kind = "Labels"; + } + + if ($limit_query_part) { + $offset_query_part = "OFFSET $offset"; + } + + // proper override_order applied above + if ($vfeed_query_part && !$ignore_vfeed_group && get_pref('VFEED_GROUP_BY_FEED', $owner_uid)) { + if (!$override_order) { + $order_by = "ttrss_feeds.title, $order_by"; + } else { + $order_by = "ttrss_feeds.title, $override_order"; + } + } + + if (!$allow_archived) { + $from_qpart = "ttrss_entries,ttrss_user_entries,ttrss_feeds$ext_tables_part"; + $feed_check_qpart = "ttrss_user_entries.feed_id = ttrss_feeds.id AND"; + + } else { + $from_qpart = "ttrss_entries$ext_tables_part,ttrss_user_entries + LEFT JOIN ttrss_feeds ON (feed_id = ttrss_feeds.id)"; + } + + if ($vfeed_query_part) + $vfeed_query_part .= "favicon_avg_color,"; + + $query = "SELECT DISTINCT + date_entered, + guid, + ttrss_entries.id,ttrss_entries.title, + updated, + label_cache, + tag_cache, + always_display_enclosures, + site_url, + note, + num_comments, + comments, + int_id, + uuid, + lang, + hide_images, + unread,feed_id,marked,published,link,last_read,orig_feed_id, + last_marked, last_published, + $vfeed_query_part + $content_query_part + author,score + FROM + $from_qpart + WHERE + $feed_check_qpart + ttrss_user_entries.ref_id = ttrss_entries.id AND + ttrss_user_entries.owner_uid = '$owner_uid' AND + $search_query_part + $filter_query_part + $view_query_part + $since_id_part + $query_strategy_part ORDER BY $order_by + $limit_query_part $offset_query_part"; + + if ($_REQUEST["debug"]) print $query; + + $result = db_query($query); + + } else { + // browsing by tag + + $select_qpart = "SELECT DISTINCT " . + "date_entered," . + "guid," . + "note," . + "ttrss_entries.id as id," . + "title," . + "updated," . + "unread," . + "feed_id," . + "orig_feed_id," . + "marked," . + "num_comments, " . + "comments, " . + "tag_cache," . + "label_cache," . + "link," . + "lang," . + "uuid," . + "last_read," . + "(SELECT hide_images FROM ttrss_feeds WHERE id = feed_id) AS hide_images," . + "last_marked, last_published, " . + $since_id_part . + $vfeed_query_part . + $content_query_part . + "score "; + + $feed_kind = "Tags"; + $all_tags = explode(",", $feed); + if ($search_mode == 'any') { + $tag_sql = "tag_name in (" . implode(", ", array_map("db_quote", $all_tags)) . ")"; + $from_qpart = " FROM ttrss_entries,ttrss_user_entries,ttrss_tags "; + $where_qpart = " WHERE " . + "ref_id = ttrss_entries.id AND " . + "ttrss_user_entries.owner_uid = $owner_uid AND " . + "post_int_id = int_id AND $tag_sql AND " . + $view_query_part . + $search_query_part . + $query_strategy_part . " ORDER BY $order_by " . + $limit_query_part; + + } else { + $i = 1; + $sub_selects = array(); + $sub_ands = array(); + foreach ($all_tags as $term) { + array_push($sub_selects, "(SELECT post_int_id from ttrss_tags WHERE tag_name = " . db_quote($term) . " AND owner_uid = $owner_uid) as A$i"); + $i++; + } + if ($i > 2) { + $x = 1; + $y = 2; + do { + array_push($sub_ands, "A$x.post_int_id = A$y.post_int_id"); + $x++; + $y++; + } while ($y < $i); + } + array_push($sub_ands, "A1.post_int_id = ttrss_user_entries.int_id and ttrss_user_entries.owner_uid = $owner_uid"); + array_push($sub_ands, "ttrss_user_entries.ref_id = ttrss_entries.id"); + $from_qpart = " FROM " . implode(", ", $sub_selects) . ", ttrss_user_entries, ttrss_entries"; + $where_qpart = " WHERE " . implode(" AND ", $sub_ands); + } + // error_log("TAG SQL: " . $tag_sql); + // $tag_sql = "tag_name = '$feed'"; DEFAULT way + + // error_log("[". $select_qpart . "][" . $from_qpart . "][" .$where_qpart . "]"); + $result = db_query($select_qpart . $from_qpart . $where_qpart); + } + + return array($result, $feed_title, $feed_site_url, $last_error, $last_updated, $search_words); + + } + + function sanitize($str, $force_remove_images = false, $owner = false, $site_url = false, $highlight_words = false, $article_id = false) { + if (!$owner) $owner = $_SESSION["uid"]; + + $res = trim($str); if (!$res) return ''; + + $charset_hack = ' + + '; + + $res = trim($res); if (!$res) return ''; + + libxml_use_internal_errors(true); + + $doc = new DOMDocument(); + $doc->loadHTML($charset_hack . $res); + $xpath = new DOMXPath($doc); + + $entries = $xpath->query('(//a[@href]|//img[@src])'); + + foreach ($entries as $entry) { + + if ($site_url) { + + if ($entry->hasAttribute('href')) { + $entry->setAttribute('href', + rewrite_relative_url($site_url, $entry->getAttribute('href'))); + + $entry->setAttribute('rel', 'noreferrer'); + } + + if ($entry->hasAttribute('src')) { + $src = rewrite_relative_url($site_url, $entry->getAttribute('src')); + + $cached_filename = CACHE_DIR . '/images/' . sha1($src) . '.png'; + + if (file_exists($cached_filename)) { + $src = SELF_URL_PATH . '/image.php?hash=' . sha1($src); + } + + $entry->setAttribute('src', $src); + } + + if ($entry->nodeName == 'img') { + if (($owner && get_pref("STRIP_IMAGES", $owner)) || + $force_remove_images || $_SESSION["bw_limit"]) { + + $p = $doc->createElement('p'); + + $a = $doc->createElement('a'); + $a->setAttribute('href', $entry->getAttribute('src')); + + $a->appendChild(new DOMText($entry->getAttribute('src'))); + $a->setAttribute('target', '_blank'); + + $p->appendChild($a); + + $entry->parentNode->replaceChild($p, $entry); + } + } + } + + if (strtolower($entry->nodeName) == "a") { + $entry->setAttribute("target", "_blank"); + } + } + + $entries = $xpath->query('//iframe'); + foreach ($entries as $entry) { + $entry->setAttribute('sandbox', 'allow-scripts'); + + } + + $allowed_elements = array('a', 'address', 'audio', 'article', 'aside', + 'b', 'bdi', 'bdo', 'big', 'blockquote', 'body', 'br', + 'caption', 'cite', 'center', 'code', 'col', 'colgroup', + 'data', 'dd', 'del', 'details', 'div', 'dl', 'font', + 'dt', 'em', 'footer', 'figure', 'figcaption', + 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'html', 'i', + 'img', 'ins', 'kbd', 'li', 'main', 'mark', 'nav', 'noscript', + 'ol', 'p', 'pre', 'q', 'ruby', 'rp', 'rt', 's', 'samp', 'section', + 'small', 'source', 'span', 'strike', 'strong', 'sub', 'summary', + 'sup', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', 'time', + 'tr', 'track', 'tt', 'u', 'ul', 'var', 'wbr', 'video' ); + + if ($_SESSION['hasSandbox']) $allowed_elements[] = 'iframe'; + + $disallowed_attributes = array('id', 'style', 'class'); + + foreach (PluginHost::getInstance()->get_hooks(PluginHost::HOOK_SANITIZE) as $plugin) { + $retval = $plugin->hook_sanitize($doc, $site_url, $allowed_elements, $disallowed_attributes, $article_id); + if (is_array($retval)) { + $doc = $retval[0]; + $allowed_elements = $retval[1]; + $disallowed_attributes = $retval[2]; + } else { + $doc = $retval; + } + } + + $doc->removeChild($doc->firstChild); //remove doctype + $doc = strip_harmful_tags($doc, $allowed_elements, $disallowed_attributes); + + if ($highlight_words) { + foreach ($highlight_words as $word) { + + // http://stackoverflow.com/questions/4081372/highlight-keywords-in-a-paragraph + + $elements = $xpath->query("//*/text()"); + + foreach ($elements as $child) { + + $fragment = $doc->createDocumentFragment(); + $text = $child->textContent; + + while (($pos = mb_stripos($text, $word)) !== false) { + $fragment->appendChild(new DomText(mb_substr($text, 0, $pos))); + $word = mb_substr($text, $pos, mb_strlen($word)); + $highlight = $doc->createElement('span'); + $highlight->appendChild(new DomText($word)); + $highlight->setAttribute('class', 'highlight'); + $fragment->appendChild($highlight); + $text = mb_substr($text, $pos + mb_strlen($word)); + } + + if (!empty($text)) $fragment->appendChild(new DomText($text)); + + $child->parentNode->replaceChild($fragment, $child); + } + } + } + + $res = $doc->saveHTML(); + + return $res; + } + + function strip_harmful_tags($doc, $allowed_elements, $disallowed_attributes) { + $xpath = new DOMXPath($doc); + $entries = $xpath->query('//*'); + + foreach ($entries as $entry) { + if (!in_array($entry->nodeName, $allowed_elements)) { + $entry->parentNode->removeChild($entry); + } + + if ($entry->hasAttributes()) { + $attrs_to_remove = array(); + + foreach ($entry->attributes as $attr) { + + if (strpos($attr->nodeName, 'on') === 0) { + array_push($attrs_to_remove, $attr); + } + + if (in_array($attr->nodeName, $disallowed_attributes)) { + array_push($attrs_to_remove, $attr); + } + } + + foreach ($attrs_to_remove as $attr) { + $entry->removeAttributeNode($attr); + } + } + } + + return $doc; + } + + function check_for_update() { + if (CHECK_FOR_NEW_VERSION && $_SESSION['access_level'] >= 10) { + $version_url = "http://tt-rss.org/version.php?ver=" . VERSION . + "&iid=" . sha1(SELF_URL_PATH); + + $version_data = @fetch_file_contents($version_url); + + if ($version_data) { + $version_data = json_decode($version_data, true); + if ($version_data && $version_data['version']) { + if (version_compare(VERSION_STATIC, $version_data['version']) == -1) { + return $version_data; + } + } + } + } + return false; + } + + function catchupArticlesById($ids, $cmode, $owner_uid = false) { + + if (!$owner_uid) $owner_uid = $_SESSION["uid"]; + if (count($ids) == 0) return; + + $tmp_ids = array(); + + foreach ($ids as $id) { + array_push($tmp_ids, "ref_id = '$id'"); + } + + $ids_qpart = join(" OR ", $tmp_ids); + + if ($cmode == 0) { + db_query("UPDATE ttrss_user_entries SET + unread = false,last_read = NOW() + WHERE ($ids_qpart) AND owner_uid = $owner_uid"); + } else if ($cmode == 1) { + db_query("UPDATE ttrss_user_entries SET + unread = true + WHERE ($ids_qpart) AND owner_uid = $owner_uid"); + } else { + db_query("UPDATE ttrss_user_entries SET + unread = NOT unread,last_read = NOW() + WHERE ($ids_qpart) AND owner_uid = $owner_uid"); + } + + /* update ccache */ + + $result = db_query("SELECT DISTINCT feed_id FROM ttrss_user_entries + WHERE ($ids_qpart) AND owner_uid = $owner_uid"); + + while ($line = db_fetch_assoc($result)) { + ccache_update($line["feed_id"], $owner_uid); + } + } + + function get_article_tags($id, $owner_uid = 0, $tag_cache = false) { + + $a_id = db_escape_string($id); + + if (!$owner_uid) $owner_uid = $_SESSION["uid"]; + + $query = "SELECT DISTINCT tag_name, + owner_uid as owner FROM + ttrss_tags WHERE post_int_id = (SELECT int_id FROM ttrss_user_entries WHERE + ref_id = '$a_id' AND owner_uid = '$owner_uid' LIMIT 1) ORDER BY tag_name"; + + $tags = array(); + + /* check cache first */ + + if ($tag_cache === false) { + $result = db_query("SELECT tag_cache FROM ttrss_user_entries + WHERE ref_id = '$id' AND owner_uid = $owner_uid"); + + $tag_cache = db_fetch_result($result, 0, "tag_cache"); + } + + if ($tag_cache) { + $tags = explode(",", $tag_cache); + } else { + + /* do it the hard way */ + + $tmp_result = db_query($query); + + while ($tmp_line = db_fetch_assoc($tmp_result)) { + array_push($tags, $tmp_line["tag_name"]); + } + + /* update the cache */ + + $tags_str = db_escape_string(join(",", $tags)); + + db_query("UPDATE ttrss_user_entries + SET tag_cache = '$tags_str' WHERE ref_id = '$id' + AND owner_uid = $owner_uid"); + } + + return $tags; + } + + function trim_array($array) { + $tmp = $array; + array_walk($tmp, 'trim'); + return $tmp; + } + + function tag_is_valid($tag) { + if ($tag == '') return false; + if (preg_match("/^[0-9]*$/", $tag)) return false; + if (mb_strlen($tag) > 250) return false; + + if (!$tag) return false; + + return true; + } + + function render_login_form() { + header('Cache-Control: public'); + + require_once "login_form.php"; + exit; + } + + function format_warning($msg, $id = "") { + return "
+ $msg
"; + } + + function format_notice($msg, $id = "") { + return "
+ $msg
"; + } + + function format_error($msg, $id = "") { + return "
+ $msg
"; + } + + function print_notice($msg) { + return print format_notice($msg); + } + + function print_warning($msg) { + return print format_warning($msg); + } + + function print_error($msg) { + return print format_error($msg); + } + + + function T_sprintf() { + $args = func_get_args(); + return vsprintf(__(array_shift($args)), $args); + } + + function format_inline_player($url, $ctype) { + + $entry = ""; + + $url = htmlspecialchars($url); + + if (strpos($ctype, "audio/") === 0) { + + if ($_SESSION["hasAudio"] && (strpos($ctype, "ogg") !== false || + $_SESSION["hasMp3"])) { + + $entry .= ""; + + } else { + + $entry .= " + + "; + } + + if ($entry) $entry .= "  " . basename($url) . ""; + + return $entry; + + } + + return ""; + +/* $filename = substr($url, strrpos($url, "/")+1); + + $entry .= " " . + $filename . " (" . $ctype . ")" . ""; */ + + } + + function format_article($id, $mark_as_read = true, $zoom_mode = false, $owner_uid = false) { + if (!$owner_uid) $owner_uid = $_SESSION["uid"]; + + $rv = array(); + + $rv['id'] = $id; + + /* we can figure out feed_id from article id anyway, why do we + * pass feed_id here? let's ignore the argument :(*/ + + $result = db_query("SELECT feed_id FROM ttrss_user_entries + WHERE ref_id = '$id'"); + + $feed_id = (int) db_fetch_result($result, 0, "feed_id"); + + $rv['feed_id'] = $feed_id; + + //if (!$zoom_mode) { print "
get_hooks(PluginHost::HOOK_RENDER_ARTICLE) as $p) { + $line = $p->hook_render_article($line); + } + + $num_comments = $line["num_comments"]; + $entry_comments = ""; + + if ($num_comments > 0) { + if ($line["comments"]) { + $comments_url = htmlspecialchars($line["comments"]); + } else { + $comments_url = htmlspecialchars($line["link"]); + } + $entry_comments = "$num_comments ". + _ngettext("comment", "comments", $num_comments).""; + + } else { + if ($line["comments"] && $line["link"] != $line["comments"]) { + $entry_comments = "".__("comments").""; + } + } + + if ($zoom_mode) { + header("Content-Type: text/html"); + $rv['content'] .= " + + Tiny Tiny RSS - ".$line["title"]."". + stylesheet_tag("css/tt-rss.css"). + stylesheet_tag("css/zoom.css"). + stylesheet_tag("css/dijit.css")." + + + + + + "; + } + + $rv['content'] .= "
"; + + $rv['content'] .= "
"; + + $entry_author = $line["author"]; + + if ($entry_author) { + $entry_author = __(" - ") . $entry_author; + } + + $parsed_updated = make_local_datetime($line["updated"], true, + $owner_uid, true); + + if (!$zoom_mode) + $rv['content'] .= "
$parsed_updated
"; + + if ($line["link"]) { + $rv['content'] .= "
" . + $line["title"] . "" . + "$entry_author
"; + } else { + $rv['content'] .= "
" . $line["title"] . "$entry_author
"; + } + + if ($zoom_mode) { + $feed_title = "". + htmlspecialchars($line["feed_title"]).""; + + $rv['content'] .= "
$feed_title
"; + + $rv['content'] .= "
$parsed_updated
"; + } + + $tags_str = format_tags_string($line["tags"], $id); + $tags_str_full = join(", ", $line["tags"]); + + if (!$tags_str_full) $tags_str_full = __("no tags"); + + if (!$entry_comments) $entry_comments = " "; # placeholder + + $rv['content'] .= "
+ Tags "; + + if (!$zoom_mode) { + $rv['content'] .= "$tags_str + (+)"; + + $rv['content'] .= "
$tags_str_full
"; + + foreach (PluginHost::getInstance()->get_hooks(PluginHost::HOOK_ARTICLE_BUTTON) as $p) { + $rv['content'] .= $p->hook_article_button($line); + } + + } else { + $tags_str = strip_tags($tags_str); + $rv['content'] .= "$tags_str"; + } + $rv['content'] .= "
"; + $rv['content'] .= "
"; + + foreach (PluginHost::getInstance()->get_hooks(PluginHost::HOOK_ARTICLE_LEFT_BUTTON) as $p) { + $rv['content'] .= $p->hook_article_left_button($line); + } + + $rv['content'] .= "$entry_comments
"; + + if ($line["orig_feed_id"]) { + + $tmp_result = db_query("SELECT * FROM ttrss_archived_feeds + WHERE id = ".$line["orig_feed_id"]); + + if (db_num_rows($tmp_result) != 0) { + + $rv['content'] .= "
"; + $rv['content'] .= __("Originally from:"); + + $rv['content'] .= " "; + + $tmp_line = db_fetch_assoc($tmp_result); + + $rv['content'] .= "" . + $tmp_line['title'] . ""; + + $rv['content'] .= " "; + + $rv['content'] .= ""; + $rv['content'] .= ""; + + $rv['content'] .= "
"; + } + } + + $rv['content'] .= "
"; + + $rv['content'] .= "
"; + if ($line['note']) { + $rv['content'] .= format_article_note($id, $line['note'], !$zoom_mode); + } + $rv['content'] .= "
"; + + if (!$line['lang']) $line['lang'] = 'en'; + + $rv['content'] .= "
"; + + $rv['content'] .= $line["content"]; + $rv['content'] .= format_article_enclosures($id, + sql_bool_to_bool($line["always_display_enclosures"]), + $line["content"], + sql_bool_to_bool($line["hide_images"])); + + $rv['content'] .= "
"; + + $rv['content'] .= "
"; + + } + + if ($zoom_mode) { + $rv['content'] .= " +
+
"; + $rv['content'] .= ""; + } + + return $rv; + + } + + function print_checkpoint($n, $s) { + $ts = microtime(true); + echo sprintf("\n", $ts - $s); + return $ts; + } + + function sanitize_tag($tag) { + $tag = trim($tag); + + $tag = mb_strtolower($tag, 'utf-8'); + + $tag = preg_replace('/[\'\"\+\>\<]/', "", $tag); + +// $tag = str_replace('"', "", $tag); +// $tag = str_replace("+", " ", $tag); + $tag = str_replace("technorati tag: ", "", $tag); + + return $tag; + } + + function get_self_url_prefix() { + if (strrpos(SELF_URL_PATH, "/") === strlen(SELF_URL_PATH)-1) { + return substr(SELF_URL_PATH, 0, strlen(SELF_URL_PATH)-1); + } else { + return SELF_URL_PATH; + } + } + + /** + * Compute the Mozilla Firefox feed adding URL from server HOST and REQUEST_URI. + * + * @return string The Mozilla Firefox feed adding URL. + */ + function add_feed_url() { + //$url_path = ($_SERVER['HTTPS'] != "on" ? 'http://' : 'https://') . $_SERVER["HTTP_HOST"] . parse_url($_SERVER["REQUEST_URI"], PHP_URL_PATH); + + $url_path = get_self_url_prefix() . + "/public.php?op=subscribe&feed_url=%s"; + return $url_path; + } // function add_feed_url + + function encrypt_password($pass, $salt = '', $mode2 = false) { + if ($salt && $mode2) { + return "MODE2:" . hash('sha256', $salt . $pass); + } else if ($salt) { + return "SHA1X:" . sha1("$salt:$pass"); + } else { + return "SHA1:" . sha1($pass); + } + } // function encrypt_password + + function load_filters($feed_id, $owner_uid, $action_id = false) { + $filters = array(); + + $cat_id = (int)getFeedCategory($feed_id); + + if ($cat_id == 0) + $null_cat_qpart = "cat_id IS NULL OR"; + else + $null_cat_qpart = ""; + + $result = db_query("SELECT * FROM ttrss_filters2 WHERE + owner_uid = $owner_uid AND enabled = true ORDER BY order_id, title"); + + $check_cats = join(",", array_merge( + getParentCategories($cat_id, $owner_uid), + array($cat_id))); + + while ($line = db_fetch_assoc($result)) { + $filter_id = $line["id"]; + + $result2 = db_query("SELECT + r.reg_exp, r.inverse, r.feed_id, r.cat_id, r.cat_filter, t.name AS type_name + FROM ttrss_filters2_rules AS r, + ttrss_filter_types AS t + WHERE + ($null_cat_qpart (cat_id IS NULL AND cat_filter = false) OR cat_id IN ($check_cats)) AND + (feed_id IS NULL OR feed_id = '$feed_id') AND + filter_type = t.id AND filter_id = '$filter_id'"); + + $rules = array(); + $actions = array(); + + while ($rule_line = db_fetch_assoc($result2)) { +# print_r($rule_line); + + $rule = array(); + $rule["reg_exp"] = $rule_line["reg_exp"]; + $rule["type"] = $rule_line["type_name"]; + $rule["inverse"] = sql_bool_to_bool($rule_line["inverse"]); + + array_push($rules, $rule); + } + + $result2 = db_query("SELECT a.action_param,t.name AS type_name + FROM ttrss_filters2_actions AS a, + ttrss_filter_actions AS t + WHERE + action_id = t.id AND filter_id = '$filter_id'"); + + while ($action_line = db_fetch_assoc($result2)) { +# print_r($action_line); + + $action = array(); + $action["type"] = $action_line["type_name"]; + $action["param"] = $action_line["action_param"]; + + array_push($actions, $action); + } + + + $filter = array(); + $filter["match_any_rule"] = sql_bool_to_bool($line["match_any_rule"]); + $filter["inverse"] = sql_bool_to_bool($line["inverse"]); + $filter["rules"] = $rules; + $filter["actions"] = $actions; + + if (count($rules) > 0 && count($actions) > 0) { + array_push($filters, $filter); + } + } + + return $filters; + } + + function get_score_pic($score) { + if ($score > 100) { + return "score_high.png"; + } else if ($score > 0) { + return "score_half_high.png"; + } else if ($score < -100) { + return "score_low.png"; + } else if ($score < 0) { + return "score_half_low.png"; + } else { + return "score_neutral.png"; + } + } + + function feed_has_icon($id) { + return is_file(ICONS_DIR . "/$id.ico") && filesize(ICONS_DIR . "/$id.ico") > 0; + } + + function init_plugins() { + PluginHost::getInstance()->load(PLUGINS, PluginHost::KIND_ALL); + + return true; + } + + function format_tags_string($tags, $id) { + if (!is_array($tags) || count($tags) == 0) { + return __("no tags"); + } else { + $maxtags = min(5, count($tags)); + + for ($i = 0; $i < $maxtags; $i++) { + $tags_str .= "" . $tags[$i] . ", "; + } + + $tags_str = mb_substr($tags_str, 0, mb_strlen($tags_str)-2); + + if (count($tags) > $maxtags) + $tags_str .= ", …"; + + return $tags_str; + } + } + + function format_article_labels($labels, $id) { + + if (!is_array($labels)) return ''; + + $labels_str = ""; + + foreach ($labels as $l) { + $labels_str .= sprintf("%s", + $l[2], $l[3], $l[1]); + } + + return $labels_str; + + } + + function format_article_note($id, $note, $allow_edit = true) { + + $str = "
+
". + ($allow_edit ? __('(edit note)') : "")."
$note
"; + + return $str; + } + + + function get_feed_category($feed_cat, $parent_cat_id = false) { + if ($parent_cat_id) { + $parent_qpart = "parent_cat = '$parent_cat_id'"; + $parent_insert = "'$parent_cat_id'"; + } else { + $parent_qpart = "parent_cat IS NULL"; + $parent_insert = "NULL"; + } + + $result = db_query( + "SELECT id FROM ttrss_feed_categories + WHERE $parent_qpart AND title = '$feed_cat' AND owner_uid = ".$_SESSION["uid"]); + + if (db_num_rows($result) == 0) { + return false; + } else { + return db_fetch_result($result, 0, "id"); + } + } + + function add_feed_category($feed_cat, $parent_cat_id = false) { + + if (!$feed_cat) return false; + + db_query("BEGIN"); + + if ($parent_cat_id) { + $parent_qpart = "parent_cat = '$parent_cat_id'"; + $parent_insert = "'$parent_cat_id'"; + } else { + $parent_qpart = "parent_cat IS NULL"; + $parent_insert = "NULL"; + } + + $feed_cat = mb_substr($feed_cat, 0, 250); + + $result = db_query( + "SELECT id FROM ttrss_feed_categories + WHERE $parent_qpart AND title = '$feed_cat' AND owner_uid = ".$_SESSION["uid"]); + + if (db_num_rows($result) == 0) { + + $result = db_query( + "INSERT INTO ttrss_feed_categories (owner_uid,title,parent_cat) + VALUES ('".$_SESSION["uid"]."', '$feed_cat', $parent_insert)"); + + db_query("COMMIT"); + + return true; + } + + return false; + } + + function getArticleFeed($id) { + $result = db_query("SELECT feed_id FROM ttrss_user_entries + WHERE ref_id = '$id' AND owner_uid = " . $_SESSION["uid"]); + + if (db_num_rows($result) != 0) { + return db_fetch_result($result, 0, "feed_id"); + } else { + return 0; + } + } + + /** + * Fixes incomplete URLs by prepending "http://". + * Also replaces feed:// with http://, and + * prepends a trailing slash if the url is a domain name only. + * + * @param string $url Possibly incomplete URL + * + * @return string Fixed URL. + */ + function fix_url($url) { + if (strpos($url, '://') === false) { + $url = 'http://' . $url; + } else if (substr($url, 0, 5) == 'feed:') { + $url = 'http:' . substr($url, 5); + } + + //prepend slash if the URL has no slash in it + // "http://www.example" -> "http://www.example/" + if (strpos($url, '/', strpos($url, ':') + 3) === false) { + $url .= '/'; + } + + if ($url != "http:///") + return $url; + else + return ''; + } + + function validate_feed_url($url) { + $parts = parse_url($url); + + return ($parts['scheme'] == 'http' || $parts['scheme'] == 'feed' || $parts['scheme'] == 'https'); + + } + + function get_article_enclosures($id) { + + $query = "SELECT * FROM ttrss_enclosures + WHERE post_id = '$id' AND content_url != ''"; + + $rv = array(); + + $result = db_query($query); + + if (db_num_rows($result) > 0) { + while ($line = db_fetch_assoc($result)) { + array_push($rv, $line); + } + } + + return $rv; + } + + function save_email_address($email) { + // FIXME: implement persistent storage of emails + + if (!$_SESSION['stored_emails']) + $_SESSION['stored_emails'] = array(); + + if (!in_array($email, $_SESSION['stored_emails'])) + array_push($_SESSION['stored_emails'], $email); + } + + + function get_feed_access_key($feed_id, $is_cat, $owner_uid = false) { + + if (!$owner_uid) $owner_uid = $_SESSION["uid"]; + + $sql_is_cat = bool_to_sql_bool($is_cat); + + $result = db_query("SELECT access_key FROM ttrss_access_keys + WHERE feed_id = '$feed_id' AND is_cat = $sql_is_cat + AND owner_uid = " . $owner_uid); + + if (db_num_rows($result) == 1) { + return db_fetch_result($result, 0, "access_key"); + } else { + $key = db_escape_string(uniqid(base_convert(rand(), 10, 36))); + + $result = db_query("INSERT INTO ttrss_access_keys + (access_key, feed_id, is_cat, owner_uid) + VALUES ('$key', '$feed_id', $sql_is_cat, '$owner_uid')"); + + return $key; + } + return false; + } + + function get_feeds_from_html($url, $content) + { + $url = fix_url($url); + $baseUrl = substr($url, 0, strrpos($url, '/') + 1); + + libxml_use_internal_errors(true); + + $doc = new DOMDocument(); + $doc->loadHTML($content); + $xpath = new DOMXPath($doc); + $entries = $xpath->query('/html/head/link[@rel="alternate" and '. + '(contains(@type,"rss") or contains(@type,"atom"))]|/html/head/link[@rel="feed"]'); + $feedUrls = array(); + foreach ($entries as $entry) { + if ($entry->hasAttribute('href')) { + $title = $entry->getAttribute('title'); + if ($title == '') { + $title = $entry->getAttribute('type'); + } + $feedUrl = rewrite_relative_url( + $baseUrl, $entry->getAttribute('href') + ); + $feedUrls[$feedUrl] = $title; + } + } + return $feedUrls; + } + + function is_html($content) { + return preg_match("/"; + + while ($line = db_fetch_assoc($result)) { + + $issel = ($line["caption"] == $value) ? "selected=\"1\"" : ""; + + print ""; + + } + +# print ""; + + print ""; + + + } + + function format_article_enclosures($id, $always_display_enclosures, + $article_content, $hide_images = false) { + + $result = get_article_enclosures($id); + $rv = ''; + + if (count($result) > 0) { + + $entries_html = array(); + $entries = array(); + $entries_inline = array(); + + foreach ($result as $line) { + + $url = $line["content_url"]; + $ctype = $line["content_type"]; + $title = $line["title"]; + + if (!$ctype) $ctype = __("unknown type"); + + $filename = substr($url, strrpos($url, "/")+1); + + $player = format_inline_player($url, $ctype); + + if ($player) array_push($entries_inline, $player); + +# $entry .= " " . +# $filename . " (" . $ctype . ")" . ""; + + $entry = "
$filename ($ctype)
"; + + array_push($entries_html, $entry); + + $entry = array(); + + $entry["type"] = $ctype; + $entry["filename"] = $filename; + $entry["url"] = $url; + $entry["title"] = $title; + + array_push($entries, $entry); + } + + if ($_SESSION['uid'] && !get_pref("STRIP_IMAGES") && !$_SESSION["bw_limit"]) { + if ($always_display_enclosures || + !preg_match("/

"; + } else { + $rv .= "

" .htmlspecialchars($entry["url"]) . "

"; + } + + if ($entry['title']) { + $rv.= "
${entry['title']}
"; + } + } + } + } + } + + if (count($entries_inline) > 0) { + $rv .= "
"; + foreach ($entries_inline as $entry) { $rv .= $entry; }; + $rv .= "
"; + } + + $rv .= ""; + } + + return $rv; + } + + function getLastArticleId() { + $result = db_query("SELECT MAX(ref_id) AS id FROM ttrss_user_entries + WHERE owner_uid = " . $_SESSION["uid"]); + + if (db_num_rows($result) == 1) { + return db_fetch_result($result, 0, "id"); + } else { + return -1; + } + } + + function build_url($parts) { + return $parts['scheme'] . "://" . $parts['host'] . $parts['path']; + } + + /** + * Converts a (possibly) relative URL to a absolute one. + * + * @param string $url Base URL (i.e. from where the document is) + * @param string $rel_url Possibly relative URL in the document + * + * @return string Absolute URL + */ + function rewrite_relative_url($url, $rel_url) { + if (strpos($rel_url, ":") !== false) { + return $rel_url; + } else if (strpos($rel_url, "://") !== false) { + return $rel_url; + } else if (strpos($rel_url, "//") === 0) { + # protocol-relative URL (rare but they exist) + return $rel_url; + } else if (strpos($rel_url, "/") === 0) + { + $parts = parse_url($url); + $parts['path'] = $rel_url; + + return build_url($parts); + + } else { + $parts = parse_url($url); + if (!isset($parts['path'])) { + $parts['path'] = '/'; + } + $dir = $parts['path']; + if (substr($dir, -1) !== '/') { + $dir = dirname($parts['path']); + $dir !== '/' && $dir .= '/'; + } + $parts['path'] = $dir . $rel_url; + + return build_url($parts); + } + } + + function sphinx_search($query, $offset = 0, $limit = 30) { + require_once 'lib/sphinxapi.php'; + + $sphinxClient = new SphinxClient(); + + $sphinxpair = explode(":", SPHINX_SERVER, 2); + + $sphinxClient->SetServer($sphinxpair[0], (int)$sphinxpair[1]); + $sphinxClient->SetConnectTimeout(1); + + $sphinxClient->SetFieldWeights(array('title' => 70, 'content' => 30, + 'feed_title' => 20)); + + $sphinxClient->SetMatchMode(SPH_MATCH_EXTENDED2); + $sphinxClient->SetRankingMode(SPH_RANK_PROXIMITY_BM25); + $sphinxClient->SetLimits($offset, $limit, 1000); + $sphinxClient->SetArrayResult(false); + $sphinxClient->SetFilter('owner_uid', array($_SESSION['uid'])); + + $result = $sphinxClient->Query($query, SPHINX_INDEX); + + $ids = array(); + + if (is_array($result['matches'])) { + foreach (array_keys($result['matches']) as $int_id) { + $ref_id = $result['matches'][$int_id]['attrs']['ref_id']; + array_push($ids, $ref_id); + } + } + + return $ids; + } + + function cleanup_tags($days = 14, $limit = 1000) { + + if (DB_TYPE == "pgsql") { + $interval_query = "date_updated < NOW() - INTERVAL '$days days'"; + } else if (DB_TYPE == "mysql") { + $interval_query = "date_updated < DATE_SUB(NOW(), INTERVAL $days DAY)"; + } + + $tags_deleted = 0; + + while ($limit > 0) { + $limit_part = 500; + + $query = "SELECT ttrss_tags.id AS id + FROM ttrss_tags, ttrss_user_entries, ttrss_entries + WHERE post_int_id = int_id AND $interval_query AND + ref_id = ttrss_entries.id AND tag_cache != '' LIMIT $limit_part"; + + $result = db_query($query); + + $ids = array(); + + while ($line = db_fetch_assoc($result)) { + array_push($ids, $line['id']); + } + + if (count($ids) > 0) { + $ids = join(",", $ids); + + $tmp_result = db_query("DELETE FROM ttrss_tags WHERE id IN ($ids)"); + $tags_deleted += db_affected_rows($tmp_result); + } else { + break; + } + + $limit -= $limit_part; + } + + return $tags_deleted; + } + + function print_user_stylesheet() { + $value = get_pref('USER_STYLESHEET'); + + if ($value) { + print ""; + } + + } + + function filter_to_sql($filter, $owner_uid) { + $query = array(); + + if (DB_TYPE == "pgsql") + $reg_qpart = "~"; + else + $reg_qpart = "REGEXP"; + + foreach ($filter["rules"] AS $rule) { + $rule['reg_exp'] = str_replace('/', '\/', $rule["reg_exp"]); + $regexp_valid = preg_match('/' . $rule['reg_exp'] . '/', + $rule['reg_exp']) !== FALSE; + + if ($regexp_valid) { + + $rule['reg_exp'] = db_escape_string($rule['reg_exp']); + + switch ($rule["type"]) { + case "title": + $qpart = "LOWER(ttrss_entries.title) $reg_qpart LOWER('". + $rule['reg_exp'] . "')"; + break; + case "content": + $qpart = "LOWER(ttrss_entries.content) $reg_qpart LOWER('". + $rule['reg_exp'] . "')"; + break; + case "both": + $qpart = "LOWER(ttrss_entries.title) $reg_qpart LOWER('". + $rule['reg_exp'] . "') OR LOWER(" . + "ttrss_entries.content) $reg_qpart LOWER('" . $rule['reg_exp'] . "')"; + break; + case "tag": + $qpart = "LOWER(ttrss_user_entries.tag_cache) $reg_qpart LOWER('". + $rule['reg_exp'] . "')"; + break; + case "link": + $qpart = "LOWER(ttrss_entries.link) $reg_qpart LOWER('". + $rule['reg_exp'] . "')"; + break; + case "author": + $qpart = "LOWER(ttrss_entries.author) $reg_qpart LOWER('". + $rule['reg_exp'] . "')"; + break; + } + + if (isset($rule['inverse'])) $qpart = "NOT ($qpart)"; + + if (isset($rule["feed_id"]) && $rule["feed_id"] > 0) { + $qpart .= " AND feed_id = " . db_escape_string($rule["feed_id"]); + } + + if (isset($rule["cat_id"])) { + + if ($rule["cat_id"] > 0) { + $children = getChildCategories($rule["cat_id"], $owner_uid); + array_push($children, $rule["cat_id"]); + + $children = join(",", $children); + + $cat_qpart = "cat_id IN ($children)"; + } else { + $cat_qpart = "cat_id IS NULL"; + } + + $qpart .= " AND $cat_qpart"; + } + + $qpart .= " AND feed_id IS NOT NULL"; + + array_push($query, "($qpart)"); + + } + } + + if (count($query) > 0) { + $fullquery = "(" . join($filter["match_any_rule"] ? "OR" : "AND", $query) . ")"; + } else { + $fullquery = "(false)"; + } + + if ($filter['inverse']) $fullquery = "(NOT $fullquery)"; + + return $fullquery; + } + + if (!function_exists('gzdecode')) { + function gzdecode($string) { // no support for 2nd argument + return file_get_contents('compress.zlib://data:who/cares;base64,'. + base64_encode($string)); + } + } + + function get_random_bytes($length) { + if (function_exists('openssl_random_pseudo_bytes')) { + return openssl_random_pseudo_bytes($length); + } else { + $output = ""; + + for ($i = 0; $i < $length; $i++) + $output .= chr(mt_rand(0, 255)); + + return $output; + } + } + + function read_stdin() { + $fp = fopen("php://stdin", "r"); + + if ($fp) { + $line = trim(fgets($fp)); + fclose($fp); + return $line; + } + + return null; + } + + function tmpdirname($path, $prefix) { + // Use PHP's tmpfile function to create a temporary + // directory name. Delete the file and keep the name. + $tempname = tempnam($path,$prefix); + if (!$tempname) + return false; + + if (!unlink($tempname)) + return false; + + return $tempname; + } + + function getFeedCategory($feed) { + $result = db_query("SELECT cat_id FROM ttrss_feeds + WHERE id = '$feed'"); + + if (db_num_rows($result) > 0) { + return db_fetch_result($result, 0, "cat_id"); + } else { + return false; + } + + } + + function implements_interface($class, $interface) { + return in_array($interface, class_implements($class)); + } + + function geturl($url, $depth = 0){ + + if ($depth == 20) return $url; + + if (!function_exists('curl_init')) + return user_error('CURL Must be installed for geturl function to work. Ask your host to enable it or uncomment extension=php_curl.dll in php.ini', E_USER_ERROR); + + $curl = curl_init(); + $header[0] = "Accept: text/xml,application/xml,application/xhtml+xml,"; + $header[0] .= "text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"; + $header[] = "Cache-Control: max-age=0"; + $header[] = "Connection: keep-alive"; + $header[] = "Keep-Alive: 300"; + $header[] = "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7"; + $header[] = "Accept-Language: en-us,en;q=0.5"; + $header[] = "Pragma: "; + + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0 Firefox/5.0'); + curl_setopt($curl, CURLOPT_HTTPHEADER, $header); + curl_setopt($curl, CURLOPT_HEADER, true); + curl_setopt($curl, CURLOPT_REFERER, $url); + curl_setopt($curl, CURLOPT_ENCODING, 'gzip,deflate'); + curl_setopt($curl, CURLOPT_AUTOREFERER, true); + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + //curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); //CURLOPT_FOLLOWLOCATION Disabled... + curl_setopt($curl, CURLOPT_TIMEOUT, 60); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false); + + if (defined('_CURL_HTTP_PROXY')) { + curl_setopt($curl, CURLOPT_PROXY, _CURL_HTTP_PROXY); + } + + if ((OPENSSL_VERSION_NUMBER >= 0x0090808f) && (OPENSSL_VERSION_NUMBER < 0x10000000)) { + curl_setopt($curl, CURLOPT_SSLVERSION, 3); + } + + $html = curl_exec($curl); + + $status = curl_getinfo($curl); + + if($status['http_code']!=200){ + if($status['http_code'] == 301 || $status['http_code'] == 302) { + curl_close($curl); + list($header) = explode("\r\n\r\n", $html, 2); + $matches = array(); + preg_match("/(Location:|URI:)[^(\n)]*/", $header, $matches); + $url = trim(str_replace($matches[1],"",$matches[0])); + $url_parsed = parse_url($url); + return (isset($url_parsed))? geturl($url, $depth + 1):''; + } + + global $fetch_last_error; + + $fetch_last_error = curl_errno($curl) . " " . curl_error($curl); + curl_close($curl); + +# $oline=''; +# foreach($status as $key=>$eline){$oline.='['.$key.']'.$eline.' ';} +# $line =$oline." \r\n ".$url."\r\n-----------------\r\n"; +# $handle = @fopen('./curl.error.log', 'a'); +# fwrite($handle, $line); + return FALSE; + } + curl_close($curl); + return $url; + } + + function get_minified_js($files) { + require_once 'lib/jshrink/Minifier.php'; + + $rv = ''; + + foreach ($files as $js) { + if (!isset($_GET['debug'])) { + $cached_file = CACHE_DIR . "/js/".basename($js).".js"; + + if (file_exists($cached_file) && + is_readable($cached_file) && + filemtime($cached_file) >= filemtime("js/$js.js")) { + + $rv .= file_get_contents($cached_file); + + } else { + $minified = JShrink\Minifier::minify(file_get_contents("js/$js.js")); + file_put_contents($cached_file, $minified); + $rv .= $minified; + } + } else { + $rv .= file_get_contents("js/$js.js"); + } + } + + return $rv; + } + + function stylesheet_tag($filename) { + $timestamp = filemtime($filename); + + return "\n"; + } + + function javascript_tag($filename) { + $query = ""; + + if (!(strpos($filename, "?") === FALSE)) { + $query = substr($filename, strpos($filename, "?")+1); + $filename = substr($filename, 0, strpos($filename, "?")); + } + + $timestamp = filemtime($filename); + + if ($query) $timestamp .= "&$query"; + + return "\n"; + } + + function calculate_dep_timestamp() { + $files = array_merge(glob("js/*.js"), glob("css/*.css")); + + $max_ts = -1; + + foreach ($files as $file) { + if (filemtime($file) > $max_ts) $max_ts = filemtime($file); + } + + return $max_ts; + } + + function T_js_decl($s1, $s2) { + if ($s1 && $s2) { + $s1 = preg_replace("/\n/", "", $s1); + $s2 = preg_replace("/\n/", "", $s2); + + $s1 = preg_replace("/\"/", "\\\"", $s1); + $s2 = preg_replace("/\"/", "\\\"", $s2); + + return "T_messages[\"$s1\"] = \"$s2\";\n"; + } + } + + function init_js_translations() { + + print 'var T_messages = new Object(); + + function __(msg) { + if (T_messages[msg]) { + return T_messages[msg]; + } else { + return msg; + } + } + + function ngettext(msg1, msg2, n) { + return __((parseInt(n) > 1) ? msg2 : msg1); + }'; + + $l10n = _get_reader(); + + for ($i = 0; $i < $l10n->total; $i++) { + $orig = $l10n->get_original_string($i); + if(strpos($orig, "\000") !== FALSE) { // Plural forms + $key = explode(chr(0), $orig); + print T_js_decl($key[0], _ngettext($key[0], $key[1], 1)); // Singular + print T_js_decl($key[1], _ngettext($key[0], $key[1], 2)); // Plural + } else { + $translation = __($orig); + print T_js_decl($orig, $translation); + } + } + } + + function label_to_feed_id($label) { + return LABEL_BASE_INDEX - 1 - abs($label); + } + + function feed_to_label_id($feed) { + return LABEL_BASE_INDEX - 1 + abs($feed); + } + + function format_libxml_error($error) { + return T_sprintf("LibXML error %s at line %d (column %d): %s", + $error->code, $error->line, $error->column, + $error->message); + } +?> -- cgit v1.2.3-54-g00ecf From 526e1d80be3fe843798c4e26964e10e25a48ae7a Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 5 Mar 2014 18:34:13 +0400 Subject: get_feeds_from_html: add rel='alternate feed' --- include/functions2.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 4e9796147..60c3fe59a 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -1779,7 +1779,7 @@ $doc->loadHTML($content); $xpath = new DOMXPath($doc); $entries = $xpath->query('/html/head/link[@rel="alternate" and '. - '(contains(@type,"rss") or contains(@type,"atom"))]|/html/head/link[@rel="feed"]'); + '(contains(@type,"rss") or contains(@type,"atom"))]|/html/head/link[@rel="feed" or @rel="alternate feed"]'); $feedUrls = array(); foreach ($entries as $entry) { if ($entry->hasAttribute('href')) { -- cgit v1.2.3-54-g00ecf From 16c48032519c3da551a5400f1221b6feaff02f2f Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 5 Mar 2014 18:49:52 +0400 Subject: Revert "get_feeds_from_html: add rel='alternate feed'" This reverts commit 526e1d80be3fe843798c4e26964e10e25a48ae7a. --- include/functions2.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 60c3fe59a..4e9796147 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -1779,7 +1779,7 @@ $doc->loadHTML($content); $xpath = new DOMXPath($doc); $entries = $xpath->query('/html/head/link[@rel="alternate" and '. - '(contains(@type,"rss") or contains(@type,"atom"))]|/html/head/link[@rel="feed" or @rel="alternate feed"]'); + '(contains(@type,"rss") or contains(@type,"atom"))]|/html/head/link[@rel="feed"]'); $feedUrls = array(); foreach ($entries as $entry) { if ($entry->hasAttribute('href')) { -- cgit v1.2.3-54-g00ecf From d1e631f301b77014b79b6b442b07b534ebe5ad7b Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Sun, 9 Mar 2014 21:48:22 +0400 Subject: generated feeds, add ts= (strtotime) start timestamp --- classes/handler/public.php | 12 ++++++++---- include/functions2.php | 10 +++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) (limited to 'include/functions2.php') diff --git a/classes/handler/public.php b/classes/handler/public.php index 1a586fe7a..30506fdad 100644 --- a/classes/handler/public.php +++ b/classes/handler/public.php @@ -3,7 +3,7 @@ class Handler_Public extends Handler { private function generate_syndicated_feed($owner_uid, $feed, $is_cat, $limit, $offset, $search, $search_mode, - $view_mode = false, $format = 'atom', $order = false, $orig_guid = false) { + $view_mode = false, $format = 'atom', $order = false, $orig_guid = false, $start_ts = false) { require_once "lib/MiniTemplator.class.php"; @@ -37,10 +37,13 @@ class Handler_Public extends Handler { break; } + //function queryFeedHeadlines($feed, $limit, $view_mode, $cat_view, $search, $search_mode, $override_order = false, $offset = 0, $owner_uid = 0, $filter = false, $since_id = 0, $include_children = false, $ignore_vfeed_group = false, $override_strategy = false, $override_vfeed = false, $start_ts = false) { + + $qfh_ret = queryFeedHeadlines($feed, 1, $view_mode, $is_cat, $search, $search_mode, $date_sort_field, $offset, $owner_uid, - false, 0, false, true); + false, 0, false, true, false, false, $start_ts); $result = $qfh_ret[0]; @@ -61,7 +64,7 @@ class Handler_Public extends Handler { $qfh_ret = queryFeedHeadlines($feed, $limit, $view_mode, $is_cat, $search, $search_mode, $date_sort_field, $offset, $owner_uid, - false, 0, false, true); + false, 0, false, true, false, false, $start_ts); $result = $qfh_ret[0]; @@ -375,6 +378,7 @@ class Handler_Public extends Handler { $search_mode = $this->dbh->escape_string($_REQUEST["smode"]); $view_mode = $this->dbh->escape_string($_REQUEST["view-mode"]); $order = $this->dbh->escape_string($_REQUEST["order"]); + $start_ts = $this->dbh->escape_string($_REQUEST["ts"]); $format = $this->dbh->escape_string($_REQUEST['format']); $orig_guid = sql_bool_to_bool($_REQUEST["orig_guid"]); @@ -397,7 +401,7 @@ class Handler_Public extends Handler { if ($owner_id) { $this->generate_syndicated_feed($owner_id, $feed, $is_cat, $limit, - $offset, $search, $search_mode, $view_mode, $format, $order, $orig_guid); + $offset, $search, $search_mode, $view_mode, $format, $order, $orig_guid, $start_ts); } else { header('HTTP/1.1 403 Forbidden'); } diff --git a/include/functions2.php b/include/functions2.php index 4e9796147..0c7a9c05b 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -382,7 +382,7 @@ return $rv; } - function queryFeedHeadlines($feed, $limit, $view_mode, $cat_view, $search, $search_mode, $override_order = false, $offset = 0, $owner_uid = 0, $filter = false, $since_id = 0, $include_children = false, $ignore_vfeed_group = false, $override_strategy = false, $override_vfeed = false) { + function queryFeedHeadlines($feed, $limit, $view_mode, $cat_view, $search, $search_mode, $override_order = false, $offset = 0, $owner_uid = 0, $filter = false, $since_id = 0, $include_children = false, $ignore_vfeed_group = false, $override_strategy = false, $override_vfeed = false, $start_ts = false) { if (!$owner_uid) $owner_uid = $_SESSION["uid"]; @@ -698,6 +698,13 @@ if ($vfeed_query_part) $vfeed_query_part .= "favicon_avg_color,"; + if ($start_ts) { + $start_ts_formatted = date("Y/m/d H:i:s", strtotime($start_ts)); + $start_ts_query_part = "updated >= '$start_ts_formatted' AND"; + } else { + $start_ts_query_part = ""; + } + $query = "SELECT DISTINCT date_entered, guid, @@ -726,6 +733,7 @@ ttrss_user_entries.ref_id = ttrss_entries.id AND ttrss_user_entries.owner_uid = '$owner_uid' AND $search_query_part + $start_ts_query_part $filter_query_part $view_query_part $since_id_part -- cgit v1.2.3-54-g00ecf From 99c19e1dcb31886e8003a0bd14416ca91491f61e Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Mon, 10 Mar 2014 02:19:26 +0400 Subject: start_ts for feeds: use date_entered for consistency --- include/functions2.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 0c7a9c05b..7c35b22cb 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -700,7 +700,7 @@ if ($start_ts) { $start_ts_formatted = date("Y/m/d H:i:s", strtotime($start_ts)); - $start_ts_query_part = "updated >= '$start_ts_formatted' AND"; + $start_ts_query_part = "date_entered >= '$start_ts_formatted' AND"; } else { $start_ts_query_part = ""; } -- cgit v1.2.3-54-g00ecf From ad593e43933d4908c361d5bc23d448ebd931eae6 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Thu, 13 Mar 2014 01:07:52 +0400 Subject: disable headlines grouping for recently read vfeed --- classes/feeds.php | 12 +++++++----- include/functions2.php | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'include/functions2.php') diff --git a/classes/feeds.php b/classes/feeds.php index ea3967911..c90b11d5d 100644 --- a/classes/feeds.php +++ b/classes/feeds.php @@ -248,6 +248,8 @@ class Feeds extends Handler_Protected { false, 0, $include_children); } + $vfeed_group_enabled = get_pref("VFEED_GROUP_BY_FEED") && $feed != -6; + if ($_REQUEST["debug"]) $timing_info = print_checkpoint("H1", $timing_info); $result = $qfh_ret[0]; @@ -421,7 +423,7 @@ class Feeds extends Handler_Protected { if (!get_pref('COMBINED_DISPLAY_MODE')) { - if (get_pref('VFEED_GROUP_BY_FEED')) { + if ($vfeed_group_enabled) { if ($feed_id != $vgroup_last_feed && $line["feed_title"]) { $cur_feed_title = $line["feed_title"]; @@ -472,7 +474,7 @@ class Feeds extends Handler_Protected { $reply['content'] .= ""; - if (!get_pref('VFEED_GROUP_BY_FEED')) { + if (!$vfeed_group_enabled) { if (@$line["feed_title"]) { $rgba = @$rgba_cache[$feed_id]; @@ -491,7 +493,7 @@ class Feeds extends Handler_Protected { $reply['content'] .= $score_pic; - if ($line["feed_title"] && !get_pref('VFEED_GROUP_BY_FEED')) { + if ($line["feed_title"] && !$vfeed_group_enabled) { $reply['content'] .= "hook_render_article_cdm($line); } - if (get_pref('VFEED_GROUP_BY_FEED') && $line["feed_title"]) { + if ($vfeed_group_enabled && $line["feed_title"]) { if ($feed_id != $vgroup_last_feed) { $cur_feed_title = $line["feed_title"]; @@ -592,7 +594,7 @@ class Feeds extends Handler_Protected { $reply['content'] .= ""; - if (!get_pref('VFEED_GROUP_BY_FEED')) { + if (!$vfeed_group_enabled) { if (@$line["feed_title"]) { $rgba = @$rgba_cache[$feed_id]; diff --git a/include/functions2.php b/include/functions2.php index 7c35b22cb..ed0963c46 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -583,6 +583,7 @@ $query_strategy_part = "unread = false AND last_read IS NOT NULL"; $vfeed_query_part = "ttrss_feeds.title AS feed_title,"; $allow_archived = true; + $ignore_vfeed_group = true; if (!$override_order) $override_order = "last_read DESC"; -- cgit v1.2.3-54-g00ecf From 31460f84fe350a04fd796763c4a444dcd10c7338 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 14 Mar 2014 14:38:51 +0400 Subject: make_init_params: add some additional information useful for reporting --- include/functions2.php | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index ed0963c46..57475183a 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -18,6 +18,11 @@ $params["bw_limit"] = (int) $_SESSION["bw_limit"]; $params["label_base_index"] = (int) LABEL_BASE_INDEX; + $params["php_platform"] = PHP_OS; + $params["php_version"] = PHP_VERSION; + + $params["sanity_checksum"] = sha1(file_get_contents("include/sanity_check.php")); + $result = db_query("SELECT MAX(id) AS mid, COUNT(*) AS nf FROM ttrss_feeds WHERE owner_uid = " . $_SESSION["uid"]); -- cgit v1.2.3-54-g00ecf From 3e8a79be6d49d06c3a11be832123f94e0f212a3e Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 14 Mar 2014 14:51:12 +0400 Subject: make_init_params: add theme --- include/functions2.php | 1 + 1 file changed, 1 insertion(+) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 57475183a..2cf4af22e 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -17,6 +17,7 @@ $params["default_view_order_by"] = get_pref("_DEFAULT_VIEW_ORDER_BY"); $params["bw_limit"] = (int) $_SESSION["bw_limit"]; $params["label_base_index"] = (int) LABEL_BASE_INDEX; + $params["theme"] = get_pref("USER_CSS_THEME", false, false); $params["php_platform"] = PHP_OS; $params["php_version"] = PHP_VERSION; -- cgit v1.2.3-54-g00ecf From 84e36b61a380d7d72fe7b78afe41825aa828eff7 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 14 Mar 2014 15:06:37 +0400 Subject: make_init_params: add plugins --- classes/pluginhost.php | 10 ++++++++++ include/functions2.php | 1 + 2 files changed, 11 insertions(+) (limited to 'include/functions2.php') diff --git a/classes/pluginhost.php b/classes/pluginhost.php index 472f20186..19f8b563b 100644 --- a/classes/pluginhost.php +++ b/classes/pluginhost.php @@ -75,6 +75,16 @@ class PluginHost { return $this->dbh; } + function get_plugin_names() { + $names = array(); + + foreach ($this->plugins as $p) { + array_push($names, get_class($p)); + } + + return $names; + } + function get_plugins() { return $this->plugins; } diff --git a/include/functions2.php b/include/functions2.php index 2cf4af22e..ca58ea6c5 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -18,6 +18,7 @@ $params["bw_limit"] = (int) $_SESSION["bw_limit"]; $params["label_base_index"] = (int) LABEL_BASE_INDEX; $params["theme"] = get_pref("USER_CSS_THEME", false, false); + $params["plugins"] = implode(", ", PluginHost::getInstance()->get_plugin_names()); $params["php_platform"] = PHP_OS; $params["php_version"] = PHP_VERSION; -- cgit v1.2.3-54-g00ecf From baaf4c3043f428ec009dd48b85e1c2d6cb8abee4 Mon Sep 17 00:00:00 2001 From: Rob Hoelz Date: Mon, 14 Apr 2014 23:18:33 -0500 Subject: Make search mechanism pluggable Currently, TinyTinyRSS can use raw SQL or the Sphinx search engine for searching. It would be nice if other search engines (such as Xapian) could be used, or if features of the underlying SQL engine (such as MySQL's FULLTEXT indexes) could be leveraged. This commit makes searching into a plugin hook, falling back to the builtin behavior if no search plugin is active. The Sphinx search behavior has been broken out into a plugin. --- classes/feeds.php | 2 +- classes/pluginhost.php | 1 + config.php-dist | 4 - include/functions2.php | 50 +- include/sanity_check.php | 5 - include/sanity_config.php | 2 +- lib/sphinxapi.php | 1691 ----------------------------------- plugins/search_sphinx/init.php | 60 ++ plugins/search_sphinx/sphinxapi.php | 1691 +++++++++++++++++++++++++++++++++++ 9 files changed, 1760 insertions(+), 1746 deletions(-) delete mode 100644 lib/sphinxapi.php create mode 100644 plugins/search_sphinx/init.php create mode 100644 plugins/search_sphinx/sphinxapi.php (limited to 'include/functions2.php') diff --git a/classes/feeds.php b/classes/feeds.php index ca470c032..89acb0d0b 100644 --- a/classes/feeds.php +++ b/classes/feeds.php @@ -1147,7 +1147,7 @@ class Feeds extends Handler_Protected { print "
"; - if (!SPHINX_ENABLED) { + if (count(PluginHost::getInstance()->get_hooks(PluginHost::HOOK_SEARCH)) == 0) { print ""; diff --git a/classes/pluginhost.php b/classes/pluginhost.php index 19f8b563b..4582c314a 100644 --- a/classes/pluginhost.php +++ b/classes/pluginhost.php @@ -39,6 +39,7 @@ class PluginHost { const HOOK_FETCH_FEED = 22; const HOOK_QUERY_HEADLINES = 23; const HOOK_HOUSE_KEEPING = 24; + const HOOK_SEARCH = 25; const KIND_ALL = 1; const KIND_SYSTEM = 2; diff --git a/config.php-dist b/config.php-dist index 999f00773..2d7a6d195 100644 --- a/config.php-dist +++ b/config.php-dist @@ -108,10 +108,6 @@ // *** Sphinx search *** // ********************* - define('SPHINX_ENABLED', false); - // Enable fulltext search using Sphinx (http://www.sphinxsearch.com) - // Please see http://tt-rss.org/wiki/SphinxSearch for more information. - define('SPHINX_SERVER', 'localhost:9312'); // Hostname:port combination for the Sphinx server. diff --git a/include/functions2.php b/include/functions2.php index ca58ea6c5..22c602362 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -397,20 +397,15 @@ $search_words = array(); if ($search) { + foreach (PluginHost::getInstance()->get_hooks(PluginHost::HOOK_SEARCH) as $plugin) { + list($search_query_part, $search_words) = $plugin->hook_search($search); + } - if (SPHINX_ENABLED) { - $ids = join(",", @sphinx_search($search, 0, 500)); - - if ($ids) - $search_query_part = "ref_id IN ($ids) AND "; - else - $search_query_part = "ref_id = -1 AND "; - - } else { + // fall back in case of no plugins + if (!$search_query_part) { list($search_query_part, $search_words) = search_to_sql($search); - $search_query_part .= " AND "; } - + $search_query_part .= " AND "; } else { $search_query_part = ""; } @@ -1994,39 +1989,6 @@ } } - function sphinx_search($query, $offset = 0, $limit = 30) { - require_once 'lib/sphinxapi.php'; - - $sphinxClient = new SphinxClient(); - - $sphinxpair = explode(":", SPHINX_SERVER, 2); - - $sphinxClient->SetServer($sphinxpair[0], (int)$sphinxpair[1]); - $sphinxClient->SetConnectTimeout(1); - - $sphinxClient->SetFieldWeights(array('title' => 70, 'content' => 30, - 'feed_title' => 20)); - - $sphinxClient->SetMatchMode(SPH_MATCH_EXTENDED2); - $sphinxClient->SetRankingMode(SPH_RANK_PROXIMITY_BM25); - $sphinxClient->SetLimits($offset, $limit, 1000); - $sphinxClient->SetArrayResult(false); - $sphinxClient->SetFilter('owner_uid', array($_SESSION['uid'])); - - $result = $sphinxClient->Query($query, SPHINX_INDEX); - - $ids = array(); - - if (is_array($result['matches'])) { - foreach (array_keys($result['matches']) as $int_id) { - $ref_id = $result['matches'][$int_id]['attrs']['ref_id']; - array_push($ids, $ref_id); - } - } - - return $ids; - } - function cleanup_tags($days = 14, $limit = 1000) { if (DB_TYPE == "pgsql") { diff --git a/include/sanity_check.php b/include/sanity_check.php index 1aa581bfc..6bec43051 100644 --- a/include/sanity_check.php +++ b/include/sanity_check.php @@ -146,11 +146,6 @@ array_push($errors, "PHP support for CURL is required for PubSubHubbub."); } - if (SPHINX_ENABLED && class_exists("SphinxClient")) { - array_push($errors, "Your PHP has a separate systemwide Sphinx client installed which conflicts with the client library used by tt-rss. Either remove the system library or disable Sphinx support."); - - } - if (!class_exists("DOMDocument")) { array_push($errors, "PHP support for DOMDocument is required, but was not found."); } diff --git a/include/sanity_config.php b/include/sanity_config.php index 99e83e0d9..76fba4b81 100644 --- a/include/sanity_config.php +++ b/include/sanity_config.php @@ -1,3 +1,3 @@ +$requred_defines = array( 'DB_TYPE', 'DB_HOST', 'DB_USER', 'DB_NAME', 'DB_PASS', 'MYSQL_CHARSET', 'SELF_URL_PATH', 'FEED_CRYPT_KEY', 'SINGLE_USER_MODE', 'SIMPLE_UPDATE_MODE', 'PHP_EXECUTABLE', 'LOCK_DIRECTORY', 'CACHE_DIR', 'ICONS_DIR', 'ICONS_URL', 'AUTH_AUTO_CREATE', 'AUTH_AUTO_LOGIN', 'FORCE_ARTICLE_PURGE', 'PUBSUBHUBBUB_HUB', 'PUBSUBHUBBUB_ENABLED', 'ENABLE_REGISTRATION', 'REG_NOTIFY_ADDRESS', 'REG_MAX_USERS', 'SESSION_COOKIE_LIFETIME', 'SESSION_CHECK_ADDRESS', 'SMTP_FROM_NAME', 'SMTP_FROM_ADDRESS', 'DIGEST_SUBJECT', 'SMTP_SERVER', 'SMTP_LOGIN', 'SMTP_PASSWORD', 'SMTP_SECURE', 'CHECK_FOR_NEW_VERSION', 'DETECT_ARTICLE_LANGUAGE', 'ENABLE_GZIP_OUTPUT', 'PLUGINS', 'LOG_DESTINATION', 'CONFIG_VERSION'); ?> diff --git a/lib/sphinxapi.php b/lib/sphinxapi.php deleted file mode 100644 index 90643d3c7..000000000 --- a/lib/sphinxapi.php +++ /dev/null @@ -1,1691 +0,0 @@ -=8 ) - { - $v = (int)$v; - return pack ( "NN", $v>>32, $v&0xFFFFFFFF ); - } - - // x32, int - if ( is_int($v) ) - return pack ( "NN", $v < 0 ? -1 : 0, $v ); - - // x32, bcmath - if ( function_exists("bcmul") ) - { - if ( bccomp ( $v, 0 ) == -1 ) - $v = bcadd ( "18446744073709551616", $v ); - $h = bcdiv ( $v, "4294967296", 0 ); - $l = bcmod ( $v, "4294967296" ); - return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit - } - - // x32, no-bcmath - $p = max(0, strlen($v) - 13); - $lo = abs((float)substr($v, $p)); - $hi = abs((float)substr($v, 0, $p)); - - $m = $lo + $hi*1316134912.0; // (10 ^ 13) % (1 << 32) = 1316134912 - $q = floor($m/4294967296.0); - $l = $m - ($q*4294967296.0); - $h = $hi*2328.0 + $q; // (10 ^ 13) / (1 << 32) = 2328 - - if ( $v<0 ) - { - if ( $l==0 ) - $h = 4294967296.0 - $h; - else - { - $h = 4294967295.0 - $h; - $l = 4294967296.0 - $l; - } - } - return pack ( "NN", $h, $l ); -} - -/// pack 64-bit unsigned -function sphPackU64 ( $v ) -{ - assert ( is_numeric($v) ); - - // x64 - if ( PHP_INT_SIZE>=8 ) - { - assert ( $v>=0 ); - - // x64, int - if ( is_int($v) ) - return pack ( "NN", $v>>32, $v&0xFFFFFFFF ); - - // x64, bcmath - if ( function_exists("bcmul") ) - { - $h = bcdiv ( $v, 4294967296, 0 ); - $l = bcmod ( $v, 4294967296 ); - return pack ( "NN", $h, $l ); - } - - // x64, no-bcmath - $p = max ( 0, strlen($v) - 13 ); - $lo = (int)substr ( $v, $p ); - $hi = (int)substr ( $v, 0, $p ); - - $m = $lo + $hi*1316134912; - $l = $m % 4294967296; - $h = $hi*2328 + (int)($m/4294967296); - - return pack ( "NN", $h, $l ); - } - - // x32, int - if ( is_int($v) ) - return pack ( "NN", 0, $v ); - - // x32, bcmath - if ( function_exists("bcmul") ) - { - $h = bcdiv ( $v, "4294967296", 0 ); - $l = bcmod ( $v, "4294967296" ); - return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit - } - - // x32, no-bcmath - $p = max(0, strlen($v) - 13); - $lo = (float)substr($v, $p); - $hi = (float)substr($v, 0, $p); - - $m = $lo + $hi*1316134912.0; - $q = floor($m / 4294967296.0); - $l = $m - ($q * 4294967296.0); - $h = $hi*2328.0 + $q; - - return pack ( "NN", $h, $l ); -} - -// unpack 64-bit unsigned -function sphUnpackU64 ( $v ) -{ - list ( $hi, $lo ) = array_values ( unpack ( "N*N*", $v ) ); - - if ( PHP_INT_SIZE>=8 ) - { - if ( $hi<0 ) $hi += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again - if ( $lo<0 ) $lo += (1<<32); - - // x64, int - if ( $hi<=2147483647 ) - return ($hi<<32) + $lo; - - // x64, bcmath - if ( function_exists("bcmul") ) - return bcadd ( $lo, bcmul ( $hi, "4294967296" ) ); - - // x64, no-bcmath - $C = 100000; - $h = ((int)($hi / $C) << 32) + (int)($lo / $C); - $l = (($hi % $C) << 32) + ($lo % $C); - if ( $l>$C ) - { - $h += (int)($l / $C); - $l = $l % $C; - } - - if ( $h==0 ) - return $l; - return sprintf ( "%d%05d", $h, $l ); - } - - // x32, int - if ( $hi==0 ) - { - if ( $lo>0 ) - return $lo; - return sprintf ( "%u", $lo ); - } - - $hi = sprintf ( "%u", $hi ); - $lo = sprintf ( "%u", $lo ); - - // x32, bcmath - if ( function_exists("bcmul") ) - return bcadd ( $lo, bcmul ( $hi, "4294967296" ) ); - - // x32, no-bcmath - $hi = (float)$hi; - $lo = (float)$lo; - - $q = floor($hi/10000000.0); - $r = $hi - $q*10000000.0; - $m = $lo + $r*4967296.0; - $mq = floor($m/10000000.0); - $l = $m - $mq*10000000.0; - $h = $q*4294967296.0 + $r*429.0 + $mq; - - $h = sprintf ( "%.0f", $h ); - $l = sprintf ( "%07.0f", $l ); - if ( $h=="0" ) - return sprintf( "%.0f", (float)$l ); - return $h . $l; -} - -// unpack 64-bit signed -function sphUnpackI64 ( $v ) -{ - list ( $hi, $lo ) = array_values ( unpack ( "N*N*", $v ) ); - - // x64 - if ( PHP_INT_SIZE>=8 ) - { - if ( $hi<0 ) $hi += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again - if ( $lo<0 ) $lo += (1<<32); - - return ($hi<<32) + $lo; - } - - // x32, int - if ( $hi==0 ) - { - if ( $lo>0 ) - return $lo; - return sprintf ( "%u", $lo ); - } - // x32, int - elseif ( $hi==-1 ) - { - if ( $lo<0 ) - return $lo; - return sprintf ( "%.0f", $lo - 4294967296.0 ); - } - - $neg = ""; - $c = 0; - if ( $hi<0 ) - { - $hi = ~$hi; - $lo = ~$lo; - $c = 1; - $neg = "-"; - } - - $hi = sprintf ( "%u", $hi ); - $lo = sprintf ( "%u", $lo ); - - // x32, bcmath - if ( function_exists("bcmul") ) - return $neg . bcadd ( bcadd ( $lo, bcmul ( $hi, "4294967296" ) ), $c ); - - // x32, no-bcmath - $hi = (float)$hi; - $lo = (float)$lo; - - $q = floor($hi/10000000.0); - $r = $hi - $q*10000000.0; - $m = $lo + $r*4967296.0; - $mq = floor($m/10000000.0); - $l = $m - $mq*10000000.0 + $c; - $h = $q*4294967296.0 + $r*429.0 + $mq; - if ( $l==10000000 ) - { - $l = 0; - $h += 1; - } - - $h = sprintf ( "%.0f", $h ); - $l = sprintf ( "%07.0f", $l ); - if ( $h=="0" ) - return $neg . sprintf( "%.0f", (float)$l ); - return $neg . $h . $l; -} - - -function sphFixUint ( $value ) -{ - if ( PHP_INT_SIZE>=8 ) - { - // x64 route, workaround broken unpack() in 5.2.2+ - if ( $value<0 ) $value += (1<<32); - return $value; - } - else - { - // x32 route, workaround php signed/unsigned braindamage - return sprintf ( "%u", $value ); - } -} - - -/// sphinx searchd client class -class SphinxClient -{ - var $_host; ///< searchd host (default is "localhost") - var $_port; ///< searchd port (default is 9312) - var $_offset; ///< how many records to seek from result-set start (default is 0) - var $_limit; ///< how many records to return from result-set starting at offset (default is 20) - var $_mode; ///< query matching mode (default is SPH_MATCH_ALL) - var $_weights; ///< per-field weights (default is 1 for all fields) - var $_sort; ///< match sorting mode (default is SPH_SORT_RELEVANCE) - var $_sortby; ///< attribute to sort by (defualt is "") - var $_min_id; ///< min ID to match (default is 0, which means no limit) - var $_max_id; ///< max ID to match (default is 0, which means no limit) - var $_filters; ///< search filters - var $_groupby; ///< group-by attribute name - var $_groupfunc; ///< group-by function (to pre-process group-by attribute value with) - var $_groupsort; ///< group-by sorting clause (to sort groups in result set with) - var $_groupdistinct;///< group-by count-distinct attribute - var $_maxmatches; ///< max matches to retrieve - var $_cutoff; ///< cutoff to stop searching at (default is 0) - var $_retrycount; ///< distributed retries count - var $_retrydelay; ///< distributed retries delay - var $_anchor; ///< geographical anchor point - var $_indexweights; ///< per-index weights - var $_ranker; ///< ranking mode (default is SPH_RANK_PROXIMITY_BM25) - var $_maxquerytime; ///< max query time, milliseconds (default is 0, do not limit) - var $_fieldweights; ///< per-field-name weights - var $_overrides; ///< per-query attribute values overrides - var $_select; ///< select-list (attributes or expressions, with optional aliases) - - var $_error; ///< last error message - var $_warning; ///< last warning message - var $_connerror; ///< connection error vs remote error flag - - var $_reqs; ///< requests array for multi-query - var $_mbenc; ///< stored mbstring encoding - var $_arrayresult; ///< whether $result["matches"] should be a hash or an array - var $_timeout; ///< connect timeout - - ///////////////////////////////////////////////////////////////////////////// - // common stuff - ///////////////////////////////////////////////////////////////////////////// - - /// create a new client object and fill defaults - function SphinxClient () - { - // per-client-object settings - $this->_host = "localhost"; - $this->_port = 9312; - $this->_path = false; - $this->_socket = false; - - // per-query settings - $this->_offset = 0; - $this->_limit = 20; - $this->_mode = SPH_MATCH_ALL; - $this->_weights = array (); - $this->_sort = SPH_SORT_RELEVANCE; - $this->_sortby = ""; - $this->_min_id = 0; - $this->_max_id = 0; - $this->_filters = array (); - $this->_groupby = ""; - $this->_groupfunc = SPH_GROUPBY_DAY; - $this->_groupsort = "@group desc"; - $this->_groupdistinct= ""; - $this->_maxmatches = 1000; - $this->_cutoff = 0; - $this->_retrycount = 0; - $this->_retrydelay = 0; - $this->_anchor = array (); - $this->_indexweights= array (); - $this->_ranker = SPH_RANK_PROXIMITY_BM25; - $this->_maxquerytime= 0; - $this->_fieldweights= array(); - $this->_overrides = array(); - $this->_select = "*"; - - $this->_error = ""; // per-reply fields (for single-query case) - $this->_warning = ""; - $this->_connerror = false; - - $this->_reqs = array (); // requests storage (for multi-query case) - $this->_mbenc = ""; - $this->_arrayresult = false; - $this->_timeout = 0; - } - - function __destruct() - { - if ( $this->_socket !== false ) - fclose ( $this->_socket ); - } - - /// get last error message (string) - function GetLastError () - { - return $this->_error; - } - - /// get last warning message (string) - function GetLastWarning () - { - return $this->_warning; - } - - /// get last error flag (to tell network connection errors from searchd errors or broken responses) - function IsConnectError() - { - return $this->_connerror; - } - - /// set searchd host name (string) and port (integer) - function SetServer ( $host, $port = 0 ) - { - assert ( is_string($host) ); - if ( $host[0] == '/') - { - $this->_path = 'unix://' . $host; - return; - } - if ( substr ( $host, 0, 7 )=="unix://" ) - { - $this->_path = $host; - return; - } - - assert ( is_int($port) ); - $this->_host = $host; - $this->_port = $port; - $this->_path = ''; - - } - - /// set server connection timeout (0 to remove) - function SetConnectTimeout ( $timeout ) - { - assert ( is_numeric($timeout) ); - $this->_timeout = $timeout; - } - - - function _Send ( $handle, $data, $length ) - { - if ( feof($handle) || fwrite ( $handle, $data, $length ) !== $length ) - { - $this->_error = 'connection unexpectedly closed (timed out?)'; - $this->_connerror = true; - return false; - } - return true; - } - - ///////////////////////////////////////////////////////////////////////////// - - /// enter mbstring workaround mode - function _MBPush () - { - $this->_mbenc = ""; - if ( ini_get ( "mbstring.func_overload" ) & 2 ) - { - $this->_mbenc = mb_internal_encoding(); - mb_internal_encoding ( "latin1" ); - } - } - - /// leave mbstring workaround mode - function _MBPop () - { - if ( $this->_mbenc ) - mb_internal_encoding ( $this->_mbenc ); - } - - /// connect to searchd server - function _Connect () - { - if ( $this->_socket!==false ) - { - // we are in persistent connection mode, so we have a socket - // however, need to check whether it's still alive - if ( !@feof ( $this->_socket ) ) - return $this->_socket; - - // force reopen - $this->_socket = false; - } - - $errno = 0; - $errstr = ""; - $this->_connerror = false; - - if ( $this->_path ) - { - $host = $this->_path; - $port = 0; - } - else - { - $host = $this->_host; - $port = $this->_port; - } - - if ( $this->_timeout<=0 ) - $fp = @fsockopen ( $host, $port, $errno, $errstr ); - else - $fp = @fsockopen ( $host, $port, $errno, $errstr, $this->_timeout ); - - if ( !$fp ) - { - if ( $this->_path ) - $location = $this->_path; - else - $location = "{$this->_host}:{$this->_port}"; - - $errstr = trim ( $errstr ); - $this->_error = "connection to $location failed (errno=$errno, msg=$errstr)"; - $this->_connerror = true; - return false; - } - - // send my version - // this is a subtle part. we must do it before (!) reading back from searchd. - // because otherwise under some conditions (reported on FreeBSD for instance) - // TCP stack could throttle write-write-read pattern because of Nagle. - if ( !$this->_Send ( $fp, pack ( "N", 1 ), 4 ) ) - { - fclose ( $fp ); - $this->_error = "failed to send client protocol version"; - return false; - } - - // check version - list(,$v) = unpack ( "N*", fread ( $fp, 4 ) ); - $v = (int)$v; - if ( $v<1 ) - { - fclose ( $fp ); - $this->_error = "expected searchd protocol version 1+, got version '$v'"; - return false; - } - - return $fp; - } - - /// get and check response packet from searchd server - function _GetResponse ( $fp, $client_ver ) - { - $response = ""; - $len = 0; - - $header = fread ( $fp, 8 ); - if ( strlen($header)==8 ) - { - list ( $status, $ver, $len ) = array_values ( unpack ( "n2a/Nb", $header ) ); - $left = $len; - while ( $left>0 && !feof($fp) ) - { - $chunk = fread ( $fp, min ( 8192, $left ) ); - if ( $chunk ) - { - $response .= $chunk; - $left -= strlen($chunk); - } - } - } - if ( $this->_socket === false ) - fclose ( $fp ); - - // check response - $read = strlen ( $response ); - if ( !$response || $read!=$len ) - { - $this->_error = $len - ? "failed to read searchd response (status=$status, ver=$ver, len=$len, read=$read)" - : "received zero-sized searchd response"; - return false; - } - - // check status - if ( $status==SEARCHD_WARNING ) - { - list(,$wlen) = unpack ( "N*", substr ( $response, 0, 4 ) ); - $this->_warning = substr ( $response, 4, $wlen ); - return substr ( $response, 4+$wlen ); - } - if ( $status==SEARCHD_ERROR ) - { - $this->_error = "searchd error: " . substr ( $response, 4 ); - return false; - } - if ( $status==SEARCHD_RETRY ) - { - $this->_error = "temporary searchd error: " . substr ( $response, 4 ); - return false; - } - if ( $status!=SEARCHD_OK ) - { - $this->_error = "unknown status code '$status'"; - return false; - } - - // check version - if ( $ver<$client_ver ) - { - $this->_warning = sprintf ( "searchd command v.%d.%d older than client's v.%d.%d, some options might not work", - $ver>>8, $ver&0xff, $client_ver>>8, $client_ver&0xff ); - } - - return $response; - } - - ///////////////////////////////////////////////////////////////////////////// - // searching - ///////////////////////////////////////////////////////////////////////////// - - /// set offset and count into result set, - /// and optionally set max-matches and cutoff limits - function SetLimits ( $offset, $limit, $max=0, $cutoff=0 ) - { - assert ( is_int($offset) ); - assert ( is_int($limit) ); - assert ( $offset>=0 ); - assert ( $limit>0 ); - assert ( $max>=0 ); - $this->_offset = $offset; - $this->_limit = $limit; - if ( $max>0 ) - $this->_maxmatches = $max; - if ( $cutoff>0 ) - $this->_cutoff = $cutoff; - } - - /// set maximum query time, in milliseconds, per-index - /// integer, 0 means "do not limit" - function SetMaxQueryTime ( $max ) - { - assert ( is_int($max) ); - assert ( $max>=0 ); - $this->_maxquerytime = $max; - } - - /// set matching mode - function SetMatchMode ( $mode ) - { - assert ( $mode==SPH_MATCH_ALL - || $mode==SPH_MATCH_ANY - || $mode==SPH_MATCH_PHRASE - || $mode==SPH_MATCH_BOOLEAN - || $mode==SPH_MATCH_EXTENDED - || $mode==SPH_MATCH_FULLSCAN - || $mode==SPH_MATCH_EXTENDED2 ); - $this->_mode = $mode; - } - - /// set ranking mode - function SetRankingMode ( $ranker ) - { - assert ( $ranker>=0 && $ranker_ranker = $ranker; - } - - /// set matches sorting mode - function SetSortMode ( $mode, $sortby="" ) - { - assert ( - $mode==SPH_SORT_RELEVANCE || - $mode==SPH_SORT_ATTR_DESC || - $mode==SPH_SORT_ATTR_ASC || - $mode==SPH_SORT_TIME_SEGMENTS || - $mode==SPH_SORT_EXTENDED || - $mode==SPH_SORT_EXPR ); - assert ( is_string($sortby) ); - assert ( $mode==SPH_SORT_RELEVANCE || strlen($sortby)>0 ); - - $this->_sort = $mode; - $this->_sortby = $sortby; - } - - /// bind per-field weights by order - /// DEPRECATED; use SetFieldWeights() instead - function SetWeights ( $weights ) - { - assert ( is_array($weights) ); - foreach ( $weights as $weight ) - assert ( is_int($weight) ); - - $this->_weights = $weights; - } - - /// bind per-field weights by name - function SetFieldWeights ( $weights ) - { - assert ( is_array($weights) ); - foreach ( $weights as $name=>$weight ) - { - assert ( is_string($name) ); - assert ( is_int($weight) ); - } - $this->_fieldweights = $weights; - } - - /// bind per-index weights by name - function SetIndexWeights ( $weights ) - { - assert ( is_array($weights) ); - foreach ( $weights as $index=>$weight ) - { - assert ( is_string($index) ); - assert ( is_int($weight) ); - } - $this->_indexweights = $weights; - } - - /// set IDs range to match - /// only match records if document ID is beetwen $min and $max (inclusive) - function SetIDRange ( $min, $max ) - { - assert ( is_numeric($min) ); - assert ( is_numeric($max) ); - assert ( $min<=$max ); - $this->_min_id = $min; - $this->_max_id = $max; - } - - /// set values set filter - /// only match records where $attribute value is in given set - function SetFilter ( $attribute, $values, $exclude=false ) - { - assert ( is_string($attribute) ); - assert ( is_array($values) ); - assert ( count($values) ); - - if ( is_array($values) && count($values) ) - { - foreach ( $values as $value ) - assert ( is_numeric($value) ); - - $this->_filters[] = array ( "type"=>SPH_FILTER_VALUES, "attr"=>$attribute, "exclude"=>$exclude, "values"=>$values ); - } - } - - /// set range filter - /// only match records if $attribute value is beetwen $min and $max (inclusive) - function SetFilterRange ( $attribute, $min, $max, $exclude=false ) - { - assert ( is_string($attribute) ); - assert ( is_numeric($min) ); - assert ( is_numeric($max) ); - assert ( $min<=$max ); - - $this->_filters[] = array ( "type"=>SPH_FILTER_RANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); - } - - /// set float range filter - /// only match records if $attribute value is beetwen $min and $max (inclusive) - function SetFilterFloatRange ( $attribute, $min, $max, $exclude=false ) - { - assert ( is_string($attribute) ); - assert ( is_float($min) ); - assert ( is_float($max) ); - assert ( $min<=$max ); - - $this->_filters[] = array ( "type"=>SPH_FILTER_FLOATRANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); - } - - /// setup anchor point for geosphere distance calculations - /// required to use @geodist in filters and sorting - /// latitude and longitude must be in radians - function SetGeoAnchor ( $attrlat, $attrlong, $lat, $long ) - { - assert ( is_string($attrlat) ); - assert ( is_string($attrlong) ); - assert ( is_float($lat) ); - assert ( is_float($long) ); - - $this->_anchor = array ( "attrlat"=>$attrlat, "attrlong"=>$attrlong, "lat"=>$lat, "long"=>$long ); - } - - /// set grouping attribute and function - function SetGroupBy ( $attribute, $func, $groupsort="@group desc" ) - { - assert ( is_string($attribute) ); - assert ( is_string($groupsort) ); - assert ( $func==SPH_GROUPBY_DAY - || $func==SPH_GROUPBY_WEEK - || $func==SPH_GROUPBY_MONTH - || $func==SPH_GROUPBY_YEAR - || $func==SPH_GROUPBY_ATTR - || $func==SPH_GROUPBY_ATTRPAIR ); - - $this->_groupby = $attribute; - $this->_groupfunc = $func; - $this->_groupsort = $groupsort; - } - - /// set count-distinct attribute for group-by queries - function SetGroupDistinct ( $attribute ) - { - assert ( is_string($attribute) ); - $this->_groupdistinct = $attribute; - } - - /// set distributed retries count and delay - function SetRetries ( $count, $delay=0 ) - { - assert ( is_int($count) && $count>=0 ); - assert ( is_int($delay) && $delay>=0 ); - $this->_retrycount = $count; - $this->_retrydelay = $delay; - } - - /// set result set format (hash or array; hash by default) - /// PHP specific; needed for group-by-MVA result sets that may contain duplicate IDs - function SetArrayResult ( $arrayresult ) - { - assert ( is_bool($arrayresult) ); - $this->_arrayresult = $arrayresult; - } - - /// set attribute values override - /// there can be only one override per attribute - /// $values must be a hash that maps document IDs to attribute values - function SetOverride ( $attrname, $attrtype, $values ) - { - assert ( is_string ( $attrname ) ); - assert ( in_array ( $attrtype, array ( SPH_ATTR_INTEGER, SPH_ATTR_TIMESTAMP, SPH_ATTR_BOOL, SPH_ATTR_FLOAT, SPH_ATTR_BIGINT ) ) ); - assert ( is_array ( $values ) ); - - $this->_overrides[$attrname] = array ( "attr"=>$attrname, "type"=>$attrtype, "values"=>$values ); - } - - /// set select-list (attributes or expressions), SQL-like syntax - function SetSelect ( $select ) - { - assert ( is_string ( $select ) ); - $this->_select = $select; - } - - ////////////////////////////////////////////////////////////////////////////// - - /// clear all filters (for multi-queries) - function ResetFilters () - { - $this->_filters = array(); - $this->_anchor = array(); - } - - /// clear groupby settings (for multi-queries) - function ResetGroupBy () - { - $this->_groupby = ""; - $this->_groupfunc = SPH_GROUPBY_DAY; - $this->_groupsort = "@group desc"; - $this->_groupdistinct= ""; - } - - /// clear all attribute value overrides (for multi-queries) - function ResetOverrides () - { - $this->_overrides = array (); - } - - ////////////////////////////////////////////////////////////////////////////// - - /// connect to searchd server, run given search query through given indexes, - /// and return the search results - function Query ( $query, $index="*", $comment="" ) - { - assert ( empty($this->_reqs) ); - - $this->AddQuery ( $query, $index, $comment ); - $results = $this->RunQueries (); - $this->_reqs = array (); // just in case it failed too early - - if ( !is_array($results) ) - return false; // probably network error; error message should be already filled - - $this->_error = $results[0]["error"]; - $this->_warning = $results[0]["warning"]; - if ( $results[0]["status"]==SEARCHD_ERROR ) - return false; - else - return $results[0]; - } - - /// helper to pack floats in network byte order - function _PackFloat ( $f ) - { - $t1 = pack ( "f", $f ); // machine order - list(,$t2) = unpack ( "L*", $t1 ); // int in machine order - return pack ( "N", $t2 ); - } - - /// add query to multi-query batch - /// returns index into results array from RunQueries() call - function AddQuery ( $query, $index="*", $comment="" ) - { - // mbstring workaround - $this->_MBPush (); - - // build request - $req = pack ( "NNNNN", $this->_offset, $this->_limit, $this->_mode, $this->_ranker, $this->_sort ); // mode and limits - $req .= pack ( "N", strlen($this->_sortby) ) . $this->_sortby; - $req .= pack ( "N", strlen($query) ) . $query; // query itself - $req .= pack ( "N", count($this->_weights) ); // weights - foreach ( $this->_weights as $weight ) - $req .= pack ( "N", (int)$weight ); - $req .= pack ( "N", strlen($index) ) . $index; // indexes - $req .= pack ( "N", 1 ); // id64 range marker - $req .= sphPackU64 ( $this->_min_id ) . sphPackU64 ( $this->_max_id ); // id64 range - - // filters - $req .= pack ( "N", count($this->_filters) ); - foreach ( $this->_filters as $filter ) - { - $req .= pack ( "N", strlen($filter["attr"]) ) . $filter["attr"]; - $req .= pack ( "N", $filter["type"] ); - switch ( $filter["type"] ) - { - case SPH_FILTER_VALUES: - $req .= pack ( "N", count($filter["values"]) ); - foreach ( $filter["values"] as $value ) - $req .= sphPackI64 ( $value ); - break; - - case SPH_FILTER_RANGE: - $req .= sphPackI64 ( $filter["min"] ) . sphPackI64 ( $filter["max"] ); - break; - - case SPH_FILTER_FLOATRANGE: - $req .= $this->_PackFloat ( $filter["min"] ) . $this->_PackFloat ( $filter["max"] ); - break; - - default: - assert ( 0 && "internal error: unhandled filter type" ); - } - $req .= pack ( "N", $filter["exclude"] ); - } - - // group-by clause, max-matches count, group-sort clause, cutoff count - $req .= pack ( "NN", $this->_groupfunc, strlen($this->_groupby) ) . $this->_groupby; - $req .= pack ( "N", $this->_maxmatches ); - $req .= pack ( "N", strlen($this->_groupsort) ) . $this->_groupsort; - $req .= pack ( "NNN", $this->_cutoff, $this->_retrycount, $this->_retrydelay ); - $req .= pack ( "N", strlen($this->_groupdistinct) ) . $this->_groupdistinct; - - // anchor point - if ( empty($this->_anchor) ) - { - $req .= pack ( "N", 0 ); - } else - { - $a =& $this->_anchor; - $req .= pack ( "N", 1 ); - $req .= pack ( "N", strlen($a["attrlat"]) ) . $a["attrlat"]; - $req .= pack ( "N", strlen($a["attrlong"]) ) . $a["attrlong"]; - $req .= $this->_PackFloat ( $a["lat"] ) . $this->_PackFloat ( $a["long"] ); - } - - // per-index weights - $req .= pack ( "N", count($this->_indexweights) ); - foreach ( $this->_indexweights as $idx=>$weight ) - $req .= pack ( "N", strlen($idx) ) . $idx . pack ( "N", $weight ); - - // max query time - $req .= pack ( "N", $this->_maxquerytime ); - - // per-field weights - $req .= pack ( "N", count($this->_fieldweights) ); - foreach ( $this->_fieldweights as $field=>$weight ) - $req .= pack ( "N", strlen($field) ) . $field . pack ( "N", $weight ); - - // comment - $req .= pack ( "N", strlen($comment) ) . $comment; - - // attribute overrides - $req .= pack ( "N", count($this->_overrides) ); - foreach ( $this->_overrides as $key => $entry ) - { - $req .= pack ( "N", strlen($entry["attr"]) ) . $entry["attr"]; - $req .= pack ( "NN", $entry["type"], count($entry["values"]) ); - foreach ( $entry["values"] as $id=>$val ) - { - assert ( is_numeric($id) ); - assert ( is_numeric($val) ); - - $req .= sphPackU64 ( $id ); - switch ( $entry["type"] ) - { - case SPH_ATTR_FLOAT: $req .= $this->_PackFloat ( $val ); break; - case SPH_ATTR_BIGINT: $req .= sphPackI64 ( $val ); break; - default: $req .= pack ( "N", $val ); break; - } - } - } - - // select-list - $req .= pack ( "N", strlen($this->_select) ) . $this->_select; - - // mbstring workaround - $this->_MBPop (); - - // store request to requests array - $this->_reqs[] = $req; - return count($this->_reqs)-1; - } - - /// connect to searchd, run queries batch, and return an array of result sets - function RunQueries () - { - if ( empty($this->_reqs) ) - { - $this->_error = "no queries defined, issue AddQuery() first"; - return false; - } - - // mbstring workaround - $this->_MBPush (); - - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop (); - return false; - } - - // send query, get response - $nreqs = count($this->_reqs); - $req = join ( "", $this->_reqs ); - $len = 8+strlen($req); - $req = pack ( "nnNNN", SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, $len, 0, $nreqs ) . $req; // add header - - if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || - !( $response = $this->_GetResponse ( $fp, VER_COMMAND_SEARCH ) ) ) - { - $this->_MBPop (); - return false; - } - - // query sent ok; we can reset reqs now - $this->_reqs = array (); - - // parse and return response - return $this->_ParseSearchResponse ( $response, $nreqs ); - } - - /// parse and return search query (or queries) response - function _ParseSearchResponse ( $response, $nreqs ) - { - $p = 0; // current position - $max = strlen($response); // max position for checks, to protect against broken responses - - $results = array (); - for ( $ires=0; $ires<$nreqs && $p<$max; $ires++ ) - { - $results[] = array(); - $result =& $results[$ires]; - - $result["error"] = ""; - $result["warning"] = ""; - - // extract status - list(,$status) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $result["status"] = $status; - if ( $status!=SEARCHD_OK ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $message = substr ( $response, $p, $len ); $p += $len; - - if ( $status==SEARCHD_WARNING ) - { - $result["warning"] = $message; - } else - { - $result["error"] = $message; - continue; - } - } - - // read schema - $fields = array (); - $attrs = array (); - - list(,$nfields) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - while ( $nfields-->0 && $p<$max ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $fields[] = substr ( $response, $p, $len ); $p += $len; - } - $result["fields"] = $fields; - - list(,$nattrs) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - while ( $nattrs-->0 && $p<$max ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $attr = substr ( $response, $p, $len ); $p += $len; - list(,$type) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $attrs[$attr] = $type; - } - $result["attrs"] = $attrs; - - // read match count - list(,$count) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - list(,$id64) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - - // read matches - $idx = -1; - while ( $count-->0 && $p<$max ) - { - // index into result array - $idx++; - - // parse document id and weight - if ( $id64 ) - { - $doc = sphUnpackU64 ( substr ( $response, $p, 8 ) ); $p += 8; - list(,$weight) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - } - else - { - list ( $doc, $weight ) = array_values ( unpack ( "N*N*", - substr ( $response, $p, 8 ) ) ); - $p += 8; - $doc = sphFixUint($doc); - } - $weight = sprintf ( "%u", $weight ); - - // create match entry - if ( $this->_arrayresult ) - $result["matches"][$idx] = array ( "id"=>$doc, "weight"=>$weight ); - else - $result["matches"][$doc]["weight"] = $weight; - - // parse and create attributes - $attrvals = array (); - foreach ( $attrs as $attr=>$type ) - { - // handle 64bit ints - if ( $type==SPH_ATTR_BIGINT ) - { - $attrvals[$attr] = sphUnpackI64 ( substr ( $response, $p, 8 ) ); $p += 8; - continue; - } - - // handle floats - if ( $type==SPH_ATTR_FLOAT ) - { - list(,$uval) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - list(,$fval) = unpack ( "f*", pack ( "L", $uval ) ); - $attrvals[$attr] = $fval; - continue; - } - - // handle everything else as unsigned ints - list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - if ( $type & SPH_ATTR_MULTI ) - { - $attrvals[$attr] = array (); - $nvalues = $val; - while ( $nvalues-->0 && $p<$max ) - { - list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $attrvals[$attr][] = sphFixUint($val); - } - } else if ( $type==SPH_ATTR_STRING ) - { - $attrvals[$attr] = substr ( $response, $p, $val ); - $p += $val; - } else - { - $attrvals[$attr] = sphFixUint($val); - } - } - - if ( $this->_arrayresult ) - $result["matches"][$idx]["attrs"] = $attrvals; - else - $result["matches"][$doc]["attrs"] = $attrvals; - } - - list ( $total, $total_found, $msecs, $words ) = - array_values ( unpack ( "N*N*N*N*", substr ( $response, $p, 16 ) ) ); - $result["total"] = sprintf ( "%u", $total ); - $result["total_found"] = sprintf ( "%u", $total_found ); - $result["time"] = sprintf ( "%.3f", $msecs/1000 ); - $p += 16; - - while ( $words-->0 && $p<$max ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $word = substr ( $response, $p, $len ); $p += $len; - list ( $docs, $hits ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; - $result["words"][$word] = array ( - "docs"=>sprintf ( "%u", $docs ), - "hits"=>sprintf ( "%u", $hits ) ); - } - } - - $this->_MBPop (); - return $results; - } - - ///////////////////////////////////////////////////////////////////////////// - // excerpts generation - ///////////////////////////////////////////////////////////////////////////// - - /// connect to searchd server, and generate exceprts (snippets) - /// of given documents for given query. returns false on failure, - /// an array of snippets on success - function BuildExcerpts ( $docs, $index, $words, $opts=array() ) - { - assert ( is_array($docs) ); - assert ( is_string($index) ); - assert ( is_string($words) ); - assert ( is_array($opts) ); - - $this->_MBPush (); - - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop(); - return false; - } - - ///////////////// - // fixup options - ///////////////// - - if ( !isset($opts["before_match"]) ) $opts["before_match"] = ""; - if ( !isset($opts["after_match"]) ) $opts["after_match"] = ""; - if ( !isset($opts["chunk_separator"]) ) $opts["chunk_separator"] = " ... "; - if ( !isset($opts["limit"]) ) $opts["limit"] = 256; - if ( !isset($opts["limit_passages"]) ) $opts["limit_passages"] = 0; - if ( !isset($opts["limit_words"]) ) $opts["limit_words"] = 0; - if ( !isset($opts["around"]) ) $opts["around"] = 5; - if ( !isset($opts["exact_phrase"]) ) $opts["exact_phrase"] = false; - if ( !isset($opts["single_passage"]) ) $opts["single_passage"] = false; - if ( !isset($opts["use_boundaries"]) ) $opts["use_boundaries"] = false; - if ( !isset($opts["weight_order"]) ) $opts["weight_order"] = false; - if ( !isset($opts["query_mode"]) ) $opts["query_mode"] = false; - if ( !isset($opts["force_all_words"]) ) $opts["force_all_words"] = false; - if ( !isset($opts["start_passage_id"]) ) $opts["start_passage_id"] = 1; - if ( !isset($opts["load_files"]) ) $opts["load_files"] = false; - if ( !isset($opts["html_strip_mode"]) ) $opts["html_strip_mode"] = "index"; - if ( !isset($opts["allow_empty"]) ) $opts["allow_empty"] = false; - if ( !isset($opts["passage_boundary"]) ) $opts["passage_boundary"] = "none"; - if ( !isset($opts["emit_zones"]) ) $opts["emit_zones"] = false; - - ///////////////// - // build request - ///////////////// - - // v.1.2 req - $flags = 1; // remove spaces - if ( $opts["exact_phrase"] ) $flags |= 2; - if ( $opts["single_passage"] ) $flags |= 4; - if ( $opts["use_boundaries"] ) $flags |= 8; - if ( $opts["weight_order"] ) $flags |= 16; - if ( $opts["query_mode"] ) $flags |= 32; - if ( $opts["force_all_words"] ) $flags |= 64; - if ( $opts["load_files"] ) $flags |= 128; - if ( $opts["allow_empty"] ) $flags |= 256; - if ( $opts["emit_zones"] ) $flags |= 512; - $req = pack ( "NN", 0, $flags ); // mode=0, flags=$flags - $req .= pack ( "N", strlen($index) ) . $index; // req index - $req .= pack ( "N", strlen($words) ) . $words; // req words - - // options - $req .= pack ( "N", strlen($opts["before_match"]) ) . $opts["before_match"]; - $req .= pack ( "N", strlen($opts["after_match"]) ) . $opts["after_match"]; - $req .= pack ( "N", strlen($opts["chunk_separator"]) ) . $opts["chunk_separator"]; - $req .= pack ( "NN", (int)$opts["limit"], (int)$opts["around"] ); - $req .= pack ( "NNN", (int)$opts["limit_passages"], (int)$opts["limit_words"], (int)$opts["start_passage_id"] ); // v.1.2 - $req .= pack ( "N", strlen($opts["html_strip_mode"]) ) . $opts["html_strip_mode"]; - $req .= pack ( "N", strlen($opts["passage_boundary"]) ) . $opts["passage_boundary"]; - - // documents - $req .= pack ( "N", count($docs) ); - foreach ( $docs as $doc ) - { - assert ( is_string($doc) ); - $req .= pack ( "N", strlen($doc) ) . $doc; - } - - //////////////////////////// - // send query, get response - //////////////////////////// - - $len = strlen($req); - $req = pack ( "nnN", SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, $len ) . $req; // add header - if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || - !( $response = $this->_GetResponse ( $fp, VER_COMMAND_EXCERPT ) ) ) - { - $this->_MBPop (); - return false; - } - - ////////////////// - // parse response - ////////////////// - - $pos = 0; - $res = array (); - $rlen = strlen($response); - for ( $i=0; $i $rlen ) - { - $this->_error = "incomplete reply"; - $this->_MBPop (); - return false; - } - $res[] = $len ? substr ( $response, $pos, $len ) : ""; - $pos += $len; - } - - $this->_MBPop (); - return $res; - } - - - ///////////////////////////////////////////////////////////////////////////// - // keyword generation - ///////////////////////////////////////////////////////////////////////////// - - /// connect to searchd server, and generate keyword list for a given query - /// returns false on failure, - /// an array of words on success - function BuildKeywords ( $query, $index, $hits ) - { - assert ( is_string($query) ); - assert ( is_string($index) ); - assert ( is_bool($hits) ); - - $this->_MBPush (); - - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop(); - return false; - } - - ///////////////// - // build request - ///////////////// - - // v.1.0 req - $req = pack ( "N", strlen($query) ) . $query; // req query - $req .= pack ( "N", strlen($index) ) . $index; // req index - $req .= pack ( "N", (int)$hits ); - - //////////////////////////// - // send query, get response - //////////////////////////// - - $len = strlen($req); - $req = pack ( "nnN", SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, $len ) . $req; // add header - if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || - !( $response = $this->_GetResponse ( $fp, VER_COMMAND_KEYWORDS ) ) ) - { - $this->_MBPop (); - return false; - } - - ////////////////// - // parse response - ////////////////// - - $pos = 0; - $res = array (); - $rlen = strlen($response); - list(,$nwords) = unpack ( "N*", substr ( $response, $pos, 4 ) ); - $pos += 4; - for ( $i=0; $i<$nwords; $i++ ) - { - list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; - $tokenized = $len ? substr ( $response, $pos, $len ) : ""; - $pos += $len; - - list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; - $normalized = $len ? substr ( $response, $pos, $len ) : ""; - $pos += $len; - - $res[] = array ( "tokenized"=>$tokenized, "normalized"=>$normalized ); - - if ( $hits ) - { - list($ndocs,$nhits) = array_values ( unpack ( "N*N*", substr ( $response, $pos, 8 ) ) ); - $pos += 8; - $res [$i]["docs"] = $ndocs; - $res [$i]["hits"] = $nhits; - } - - if ( $pos > $rlen ) - { - $this->_error = "incomplete reply"; - $this->_MBPop (); - return false; - } - } - - $this->_MBPop (); - return $res; - } - - function EscapeString ( $string ) - { - $from = array ( '\\', '(',')','|','-','!','@','~','"','&', '/', '^', '$', '=' ); - $to = array ( '\\\\', '\(','\)','\|','\-','\!','\@','\~','\"', '\&', '\/', '\^', '\$', '\=' ); - - return str_replace ( $from, $to, $string ); - } - - ///////////////////////////////////////////////////////////////////////////// - // attribute updates - ///////////////////////////////////////////////////////////////////////////// - - /// batch update given attributes in given rows in given indexes - /// returns amount of updated documents (0 or more) on success, or -1 on failure - function UpdateAttributes ( $index, $attrs, $values, $mva=false ) - { - // verify everything - assert ( is_string($index) ); - assert ( is_bool($mva) ); - - assert ( is_array($attrs) ); - foreach ( $attrs as $attr ) - assert ( is_string($attr) ); - - assert ( is_array($values) ); - foreach ( $values as $id=>$entry ) - { - assert ( is_numeric($id) ); - assert ( is_array($entry) ); - assert ( count($entry)==count($attrs) ); - foreach ( $entry as $v ) - { - if ( $mva ) - { - assert ( is_array($v) ); - foreach ( $v as $vv ) - assert ( is_int($vv) ); - } else - assert ( is_int($v) ); - } - } - - // build request - $this->_MBPush (); - $req = pack ( "N", strlen($index) ) . $index; - - $req .= pack ( "N", count($attrs) ); - foreach ( $attrs as $attr ) - { - $req .= pack ( "N", strlen($attr) ) . $attr; - $req .= pack ( "N", $mva ? 1 : 0 ); - } - - $req .= pack ( "N", count($values) ); - foreach ( $values as $id=>$entry ) - { - $req .= sphPackU64 ( $id ); - foreach ( $entry as $v ) - { - $req .= pack ( "N", $mva ? count($v) : $v ); - if ( $mva ) - foreach ( $v as $vv ) - $req .= pack ( "N", $vv ); - } - } - - // connect, send query, get response - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop (); - return -1; - } - - $len = strlen($req); - $req = pack ( "nnN", SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, $len ) . $req; // add header - if ( !$this->_Send ( $fp, $req, $len+8 ) ) - { - $this->_MBPop (); - return -1; - } - - if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_UPDATE ) )) - { - $this->_MBPop (); - return -1; - } - - // parse response - list(,$updated) = unpack ( "N*", substr ( $response, 0, 4 ) ); - $this->_MBPop (); - return $updated; - } - - ///////////////////////////////////////////////////////////////////////////// - // persistent connections - ///////////////////////////////////////////////////////////////////////////// - - function Open() - { - if ( $this->_socket !== false ) - { - $this->_error = 'already connected'; - return false; - } - if ( !$fp = $this->_Connect() ) - return false; - - // command, command version = 0, body length = 4, body = 1 - $req = pack ( "nnNN", SEARCHD_COMMAND_PERSIST, 0, 4, 1 ); - if ( !$this->_Send ( $fp, $req, 12 ) ) - return false; - - $this->_socket = $fp; - return true; - } - - function Close() - { - if ( $this->_socket === false ) - { - $this->_error = 'not connected'; - return false; - } - - fclose ( $this->_socket ); - $this->_socket = false; - - return true; - } - - ////////////////////////////////////////////////////////////////////////// - // status - ////////////////////////////////////////////////////////////////////////// - - function Status () - { - $this->_MBPush (); - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop(); - return false; - } - - $req = pack ( "nnNN", SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, 1 ); // len=4, body=1 - if ( !( $this->_Send ( $fp, $req, 12 ) ) || - !( $response = $this->_GetResponse ( $fp, VER_COMMAND_STATUS ) ) ) - { - $this->_MBPop (); - return false; - } - - $res = substr ( $response, 4 ); // just ignore length, error handling, etc - $p = 0; - list ( $rows, $cols ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; - - $res = array(); - for ( $i=0; $i<$rows; $i++ ) - for ( $j=0; $j<$cols; $j++ ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $res[$i][] = substr ( $response, $p, $len ); $p += $len; - } - - $this->_MBPop (); - return $res; - } - - ////////////////////////////////////////////////////////////////////////// - // flush - ////////////////////////////////////////////////////////////////////////// - - function FlushAttributes () - { - $this->_MBPush (); - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop(); - return -1; - } - - $req = pack ( "nnN", SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0 ); // len=0 - if ( !( $this->_Send ( $fp, $req, 8 ) ) || - !( $response = $this->_GetResponse ( $fp, VER_COMMAND_FLUSHATTRS ) ) ) - { - $this->_MBPop (); - return -1; - } - - $tag = -1; - if ( strlen($response)==4 ) - list(,$tag) = unpack ( "N*", $response ); - else - $this->_error = "unexpected response length"; - - $this->_MBPop (); - return $tag; - } -} - -// -// $Id: sphinxapi.php 2758 2011-04-04 11:10:44Z kevg $ -// diff --git a/plugins/search_sphinx/init.php b/plugins/search_sphinx/init.php new file mode 100644 index 000000000..f2877e44d --- /dev/null +++ b/plugins/search_sphinx/init.php @@ -0,0 +1,60 @@ +add_hook($host::HOOK_SEARCH, $this); + + require_once __DIR__ . "/sphinxapi.php"; + } + + function hook_search($search) { + $offset = 0; + $limit = 500; + + $sphinxClient = new SphinxClient(); + + $sphinxpair = explode(":", SPHINX_SERVER, 2); + + $sphinxClient->SetServer($sphinxpair[0], (int)$sphinxpair[1]); + $sphinxClient->SetConnectTimeout(1); + + $sphinxClient->SetFieldWeights(array('title' => 70, 'content' => 30, + 'feed_title' => 20)); + + $sphinxClient->SetMatchMode(SPH_MATCH_EXTENDED2); + $sphinxClient->SetRankingMode(SPH_RANK_PROXIMITY_BM25); + $sphinxClient->SetLimits($offset, $limit, 1000); + $sphinxClient->SetArrayResult(false); + $sphinxClient->SetFilter('owner_uid', array($_SESSION['uid'])); + + $result = $sphinxClient->Query($search, SPHINX_INDEX); + + $ids = array(); + + if (is_array($result['matches'])) { + foreach (array_keys($result['matches']) as $int_id) { + $ref_id = $result['matches'][$int_id]['attrs']['ref_id']; + array_push($ids, $ref_id); + } + } + + $ids = join(",", $ids); + + if ($ids) + return array("ref_id IN ($ids)", array()); + else + return array("ref_id = -1", array()); + } + + function api_version() { + return 2; + } +} +?> diff --git a/plugins/search_sphinx/sphinxapi.php b/plugins/search_sphinx/sphinxapi.php new file mode 100644 index 000000000..90643d3c7 --- /dev/null +++ b/plugins/search_sphinx/sphinxapi.php @@ -0,0 +1,1691 @@ +=8 ) + { + $v = (int)$v; + return pack ( "NN", $v>>32, $v&0xFFFFFFFF ); + } + + // x32, int + if ( is_int($v) ) + return pack ( "NN", $v < 0 ? -1 : 0, $v ); + + // x32, bcmath + if ( function_exists("bcmul") ) + { + if ( bccomp ( $v, 0 ) == -1 ) + $v = bcadd ( "18446744073709551616", $v ); + $h = bcdiv ( $v, "4294967296", 0 ); + $l = bcmod ( $v, "4294967296" ); + return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit + } + + // x32, no-bcmath + $p = max(0, strlen($v) - 13); + $lo = abs((float)substr($v, $p)); + $hi = abs((float)substr($v, 0, $p)); + + $m = $lo + $hi*1316134912.0; // (10 ^ 13) % (1 << 32) = 1316134912 + $q = floor($m/4294967296.0); + $l = $m - ($q*4294967296.0); + $h = $hi*2328.0 + $q; // (10 ^ 13) / (1 << 32) = 2328 + + if ( $v<0 ) + { + if ( $l==0 ) + $h = 4294967296.0 - $h; + else + { + $h = 4294967295.0 - $h; + $l = 4294967296.0 - $l; + } + } + return pack ( "NN", $h, $l ); +} + +/// pack 64-bit unsigned +function sphPackU64 ( $v ) +{ + assert ( is_numeric($v) ); + + // x64 + if ( PHP_INT_SIZE>=8 ) + { + assert ( $v>=0 ); + + // x64, int + if ( is_int($v) ) + return pack ( "NN", $v>>32, $v&0xFFFFFFFF ); + + // x64, bcmath + if ( function_exists("bcmul") ) + { + $h = bcdiv ( $v, 4294967296, 0 ); + $l = bcmod ( $v, 4294967296 ); + return pack ( "NN", $h, $l ); + } + + // x64, no-bcmath + $p = max ( 0, strlen($v) - 13 ); + $lo = (int)substr ( $v, $p ); + $hi = (int)substr ( $v, 0, $p ); + + $m = $lo + $hi*1316134912; + $l = $m % 4294967296; + $h = $hi*2328 + (int)($m/4294967296); + + return pack ( "NN", $h, $l ); + } + + // x32, int + if ( is_int($v) ) + return pack ( "NN", 0, $v ); + + // x32, bcmath + if ( function_exists("bcmul") ) + { + $h = bcdiv ( $v, "4294967296", 0 ); + $l = bcmod ( $v, "4294967296" ); + return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit + } + + // x32, no-bcmath + $p = max(0, strlen($v) - 13); + $lo = (float)substr($v, $p); + $hi = (float)substr($v, 0, $p); + + $m = $lo + $hi*1316134912.0; + $q = floor($m / 4294967296.0); + $l = $m - ($q * 4294967296.0); + $h = $hi*2328.0 + $q; + + return pack ( "NN", $h, $l ); +} + +// unpack 64-bit unsigned +function sphUnpackU64 ( $v ) +{ + list ( $hi, $lo ) = array_values ( unpack ( "N*N*", $v ) ); + + if ( PHP_INT_SIZE>=8 ) + { + if ( $hi<0 ) $hi += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again + if ( $lo<0 ) $lo += (1<<32); + + // x64, int + if ( $hi<=2147483647 ) + return ($hi<<32) + $lo; + + // x64, bcmath + if ( function_exists("bcmul") ) + return bcadd ( $lo, bcmul ( $hi, "4294967296" ) ); + + // x64, no-bcmath + $C = 100000; + $h = ((int)($hi / $C) << 32) + (int)($lo / $C); + $l = (($hi % $C) << 32) + ($lo % $C); + if ( $l>$C ) + { + $h += (int)($l / $C); + $l = $l % $C; + } + + if ( $h==0 ) + return $l; + return sprintf ( "%d%05d", $h, $l ); + } + + // x32, int + if ( $hi==0 ) + { + if ( $lo>0 ) + return $lo; + return sprintf ( "%u", $lo ); + } + + $hi = sprintf ( "%u", $hi ); + $lo = sprintf ( "%u", $lo ); + + // x32, bcmath + if ( function_exists("bcmul") ) + return bcadd ( $lo, bcmul ( $hi, "4294967296" ) ); + + // x32, no-bcmath + $hi = (float)$hi; + $lo = (float)$lo; + + $q = floor($hi/10000000.0); + $r = $hi - $q*10000000.0; + $m = $lo + $r*4967296.0; + $mq = floor($m/10000000.0); + $l = $m - $mq*10000000.0; + $h = $q*4294967296.0 + $r*429.0 + $mq; + + $h = sprintf ( "%.0f", $h ); + $l = sprintf ( "%07.0f", $l ); + if ( $h=="0" ) + return sprintf( "%.0f", (float)$l ); + return $h . $l; +} + +// unpack 64-bit signed +function sphUnpackI64 ( $v ) +{ + list ( $hi, $lo ) = array_values ( unpack ( "N*N*", $v ) ); + + // x64 + if ( PHP_INT_SIZE>=8 ) + { + if ( $hi<0 ) $hi += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again + if ( $lo<0 ) $lo += (1<<32); + + return ($hi<<32) + $lo; + } + + // x32, int + if ( $hi==0 ) + { + if ( $lo>0 ) + return $lo; + return sprintf ( "%u", $lo ); + } + // x32, int + elseif ( $hi==-1 ) + { + if ( $lo<0 ) + return $lo; + return sprintf ( "%.0f", $lo - 4294967296.0 ); + } + + $neg = ""; + $c = 0; + if ( $hi<0 ) + { + $hi = ~$hi; + $lo = ~$lo; + $c = 1; + $neg = "-"; + } + + $hi = sprintf ( "%u", $hi ); + $lo = sprintf ( "%u", $lo ); + + // x32, bcmath + if ( function_exists("bcmul") ) + return $neg . bcadd ( bcadd ( $lo, bcmul ( $hi, "4294967296" ) ), $c ); + + // x32, no-bcmath + $hi = (float)$hi; + $lo = (float)$lo; + + $q = floor($hi/10000000.0); + $r = $hi - $q*10000000.0; + $m = $lo + $r*4967296.0; + $mq = floor($m/10000000.0); + $l = $m - $mq*10000000.0 + $c; + $h = $q*4294967296.0 + $r*429.0 + $mq; + if ( $l==10000000 ) + { + $l = 0; + $h += 1; + } + + $h = sprintf ( "%.0f", $h ); + $l = sprintf ( "%07.0f", $l ); + if ( $h=="0" ) + return $neg . sprintf( "%.0f", (float)$l ); + return $neg . $h . $l; +} + + +function sphFixUint ( $value ) +{ + if ( PHP_INT_SIZE>=8 ) + { + // x64 route, workaround broken unpack() in 5.2.2+ + if ( $value<0 ) $value += (1<<32); + return $value; + } + else + { + // x32 route, workaround php signed/unsigned braindamage + return sprintf ( "%u", $value ); + } +} + + +/// sphinx searchd client class +class SphinxClient +{ + var $_host; ///< searchd host (default is "localhost") + var $_port; ///< searchd port (default is 9312) + var $_offset; ///< how many records to seek from result-set start (default is 0) + var $_limit; ///< how many records to return from result-set starting at offset (default is 20) + var $_mode; ///< query matching mode (default is SPH_MATCH_ALL) + var $_weights; ///< per-field weights (default is 1 for all fields) + var $_sort; ///< match sorting mode (default is SPH_SORT_RELEVANCE) + var $_sortby; ///< attribute to sort by (defualt is "") + var $_min_id; ///< min ID to match (default is 0, which means no limit) + var $_max_id; ///< max ID to match (default is 0, which means no limit) + var $_filters; ///< search filters + var $_groupby; ///< group-by attribute name + var $_groupfunc; ///< group-by function (to pre-process group-by attribute value with) + var $_groupsort; ///< group-by sorting clause (to sort groups in result set with) + var $_groupdistinct;///< group-by count-distinct attribute + var $_maxmatches; ///< max matches to retrieve + var $_cutoff; ///< cutoff to stop searching at (default is 0) + var $_retrycount; ///< distributed retries count + var $_retrydelay; ///< distributed retries delay + var $_anchor; ///< geographical anchor point + var $_indexweights; ///< per-index weights + var $_ranker; ///< ranking mode (default is SPH_RANK_PROXIMITY_BM25) + var $_maxquerytime; ///< max query time, milliseconds (default is 0, do not limit) + var $_fieldweights; ///< per-field-name weights + var $_overrides; ///< per-query attribute values overrides + var $_select; ///< select-list (attributes or expressions, with optional aliases) + + var $_error; ///< last error message + var $_warning; ///< last warning message + var $_connerror; ///< connection error vs remote error flag + + var $_reqs; ///< requests array for multi-query + var $_mbenc; ///< stored mbstring encoding + var $_arrayresult; ///< whether $result["matches"] should be a hash or an array + var $_timeout; ///< connect timeout + + ///////////////////////////////////////////////////////////////////////////// + // common stuff + ///////////////////////////////////////////////////////////////////////////// + + /// create a new client object and fill defaults + function SphinxClient () + { + // per-client-object settings + $this->_host = "localhost"; + $this->_port = 9312; + $this->_path = false; + $this->_socket = false; + + // per-query settings + $this->_offset = 0; + $this->_limit = 20; + $this->_mode = SPH_MATCH_ALL; + $this->_weights = array (); + $this->_sort = SPH_SORT_RELEVANCE; + $this->_sortby = ""; + $this->_min_id = 0; + $this->_max_id = 0; + $this->_filters = array (); + $this->_groupby = ""; + $this->_groupfunc = SPH_GROUPBY_DAY; + $this->_groupsort = "@group desc"; + $this->_groupdistinct= ""; + $this->_maxmatches = 1000; + $this->_cutoff = 0; + $this->_retrycount = 0; + $this->_retrydelay = 0; + $this->_anchor = array (); + $this->_indexweights= array (); + $this->_ranker = SPH_RANK_PROXIMITY_BM25; + $this->_maxquerytime= 0; + $this->_fieldweights= array(); + $this->_overrides = array(); + $this->_select = "*"; + + $this->_error = ""; // per-reply fields (for single-query case) + $this->_warning = ""; + $this->_connerror = false; + + $this->_reqs = array (); // requests storage (for multi-query case) + $this->_mbenc = ""; + $this->_arrayresult = false; + $this->_timeout = 0; + } + + function __destruct() + { + if ( $this->_socket !== false ) + fclose ( $this->_socket ); + } + + /// get last error message (string) + function GetLastError () + { + return $this->_error; + } + + /// get last warning message (string) + function GetLastWarning () + { + return $this->_warning; + } + + /// get last error flag (to tell network connection errors from searchd errors or broken responses) + function IsConnectError() + { + return $this->_connerror; + } + + /// set searchd host name (string) and port (integer) + function SetServer ( $host, $port = 0 ) + { + assert ( is_string($host) ); + if ( $host[0] == '/') + { + $this->_path = 'unix://' . $host; + return; + } + if ( substr ( $host, 0, 7 )=="unix://" ) + { + $this->_path = $host; + return; + } + + assert ( is_int($port) ); + $this->_host = $host; + $this->_port = $port; + $this->_path = ''; + + } + + /// set server connection timeout (0 to remove) + function SetConnectTimeout ( $timeout ) + { + assert ( is_numeric($timeout) ); + $this->_timeout = $timeout; + } + + + function _Send ( $handle, $data, $length ) + { + if ( feof($handle) || fwrite ( $handle, $data, $length ) !== $length ) + { + $this->_error = 'connection unexpectedly closed (timed out?)'; + $this->_connerror = true; + return false; + } + return true; + } + + ///////////////////////////////////////////////////////////////////////////// + + /// enter mbstring workaround mode + function _MBPush () + { + $this->_mbenc = ""; + if ( ini_get ( "mbstring.func_overload" ) & 2 ) + { + $this->_mbenc = mb_internal_encoding(); + mb_internal_encoding ( "latin1" ); + } + } + + /// leave mbstring workaround mode + function _MBPop () + { + if ( $this->_mbenc ) + mb_internal_encoding ( $this->_mbenc ); + } + + /// connect to searchd server + function _Connect () + { + if ( $this->_socket!==false ) + { + // we are in persistent connection mode, so we have a socket + // however, need to check whether it's still alive + if ( !@feof ( $this->_socket ) ) + return $this->_socket; + + // force reopen + $this->_socket = false; + } + + $errno = 0; + $errstr = ""; + $this->_connerror = false; + + if ( $this->_path ) + { + $host = $this->_path; + $port = 0; + } + else + { + $host = $this->_host; + $port = $this->_port; + } + + if ( $this->_timeout<=0 ) + $fp = @fsockopen ( $host, $port, $errno, $errstr ); + else + $fp = @fsockopen ( $host, $port, $errno, $errstr, $this->_timeout ); + + if ( !$fp ) + { + if ( $this->_path ) + $location = $this->_path; + else + $location = "{$this->_host}:{$this->_port}"; + + $errstr = trim ( $errstr ); + $this->_error = "connection to $location failed (errno=$errno, msg=$errstr)"; + $this->_connerror = true; + return false; + } + + // send my version + // this is a subtle part. we must do it before (!) reading back from searchd. + // because otherwise under some conditions (reported on FreeBSD for instance) + // TCP stack could throttle write-write-read pattern because of Nagle. + if ( !$this->_Send ( $fp, pack ( "N", 1 ), 4 ) ) + { + fclose ( $fp ); + $this->_error = "failed to send client protocol version"; + return false; + } + + // check version + list(,$v) = unpack ( "N*", fread ( $fp, 4 ) ); + $v = (int)$v; + if ( $v<1 ) + { + fclose ( $fp ); + $this->_error = "expected searchd protocol version 1+, got version '$v'"; + return false; + } + + return $fp; + } + + /// get and check response packet from searchd server + function _GetResponse ( $fp, $client_ver ) + { + $response = ""; + $len = 0; + + $header = fread ( $fp, 8 ); + if ( strlen($header)==8 ) + { + list ( $status, $ver, $len ) = array_values ( unpack ( "n2a/Nb", $header ) ); + $left = $len; + while ( $left>0 && !feof($fp) ) + { + $chunk = fread ( $fp, min ( 8192, $left ) ); + if ( $chunk ) + { + $response .= $chunk; + $left -= strlen($chunk); + } + } + } + if ( $this->_socket === false ) + fclose ( $fp ); + + // check response + $read = strlen ( $response ); + if ( !$response || $read!=$len ) + { + $this->_error = $len + ? "failed to read searchd response (status=$status, ver=$ver, len=$len, read=$read)" + : "received zero-sized searchd response"; + return false; + } + + // check status + if ( $status==SEARCHD_WARNING ) + { + list(,$wlen) = unpack ( "N*", substr ( $response, 0, 4 ) ); + $this->_warning = substr ( $response, 4, $wlen ); + return substr ( $response, 4+$wlen ); + } + if ( $status==SEARCHD_ERROR ) + { + $this->_error = "searchd error: " . substr ( $response, 4 ); + return false; + } + if ( $status==SEARCHD_RETRY ) + { + $this->_error = "temporary searchd error: " . substr ( $response, 4 ); + return false; + } + if ( $status!=SEARCHD_OK ) + { + $this->_error = "unknown status code '$status'"; + return false; + } + + // check version + if ( $ver<$client_ver ) + { + $this->_warning = sprintf ( "searchd command v.%d.%d older than client's v.%d.%d, some options might not work", + $ver>>8, $ver&0xff, $client_ver>>8, $client_ver&0xff ); + } + + return $response; + } + + ///////////////////////////////////////////////////////////////////////////// + // searching + ///////////////////////////////////////////////////////////////////////////// + + /// set offset and count into result set, + /// and optionally set max-matches and cutoff limits + function SetLimits ( $offset, $limit, $max=0, $cutoff=0 ) + { + assert ( is_int($offset) ); + assert ( is_int($limit) ); + assert ( $offset>=0 ); + assert ( $limit>0 ); + assert ( $max>=0 ); + $this->_offset = $offset; + $this->_limit = $limit; + if ( $max>0 ) + $this->_maxmatches = $max; + if ( $cutoff>0 ) + $this->_cutoff = $cutoff; + } + + /// set maximum query time, in milliseconds, per-index + /// integer, 0 means "do not limit" + function SetMaxQueryTime ( $max ) + { + assert ( is_int($max) ); + assert ( $max>=0 ); + $this->_maxquerytime = $max; + } + + /// set matching mode + function SetMatchMode ( $mode ) + { + assert ( $mode==SPH_MATCH_ALL + || $mode==SPH_MATCH_ANY + || $mode==SPH_MATCH_PHRASE + || $mode==SPH_MATCH_BOOLEAN + || $mode==SPH_MATCH_EXTENDED + || $mode==SPH_MATCH_FULLSCAN + || $mode==SPH_MATCH_EXTENDED2 ); + $this->_mode = $mode; + } + + /// set ranking mode + function SetRankingMode ( $ranker ) + { + assert ( $ranker>=0 && $ranker_ranker = $ranker; + } + + /// set matches sorting mode + function SetSortMode ( $mode, $sortby="" ) + { + assert ( + $mode==SPH_SORT_RELEVANCE || + $mode==SPH_SORT_ATTR_DESC || + $mode==SPH_SORT_ATTR_ASC || + $mode==SPH_SORT_TIME_SEGMENTS || + $mode==SPH_SORT_EXTENDED || + $mode==SPH_SORT_EXPR ); + assert ( is_string($sortby) ); + assert ( $mode==SPH_SORT_RELEVANCE || strlen($sortby)>0 ); + + $this->_sort = $mode; + $this->_sortby = $sortby; + } + + /// bind per-field weights by order + /// DEPRECATED; use SetFieldWeights() instead + function SetWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $weight ) + assert ( is_int($weight) ); + + $this->_weights = $weights; + } + + /// bind per-field weights by name + function SetFieldWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $name=>$weight ) + { + assert ( is_string($name) ); + assert ( is_int($weight) ); + } + $this->_fieldweights = $weights; + } + + /// bind per-index weights by name + function SetIndexWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $index=>$weight ) + { + assert ( is_string($index) ); + assert ( is_int($weight) ); + } + $this->_indexweights = $weights; + } + + /// set IDs range to match + /// only match records if document ID is beetwen $min and $max (inclusive) + function SetIDRange ( $min, $max ) + { + assert ( is_numeric($min) ); + assert ( is_numeric($max) ); + assert ( $min<=$max ); + $this->_min_id = $min; + $this->_max_id = $max; + } + + /// set values set filter + /// only match records where $attribute value is in given set + function SetFilter ( $attribute, $values, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_array($values) ); + assert ( count($values) ); + + if ( is_array($values) && count($values) ) + { + foreach ( $values as $value ) + assert ( is_numeric($value) ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_VALUES, "attr"=>$attribute, "exclude"=>$exclude, "values"=>$values ); + } + } + + /// set range filter + /// only match records if $attribute value is beetwen $min and $max (inclusive) + function SetFilterRange ( $attribute, $min, $max, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_numeric($min) ); + assert ( is_numeric($max) ); + assert ( $min<=$max ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_RANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); + } + + /// set float range filter + /// only match records if $attribute value is beetwen $min and $max (inclusive) + function SetFilterFloatRange ( $attribute, $min, $max, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_float($min) ); + assert ( is_float($max) ); + assert ( $min<=$max ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_FLOATRANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); + } + + /// setup anchor point for geosphere distance calculations + /// required to use @geodist in filters and sorting + /// latitude and longitude must be in radians + function SetGeoAnchor ( $attrlat, $attrlong, $lat, $long ) + { + assert ( is_string($attrlat) ); + assert ( is_string($attrlong) ); + assert ( is_float($lat) ); + assert ( is_float($long) ); + + $this->_anchor = array ( "attrlat"=>$attrlat, "attrlong"=>$attrlong, "lat"=>$lat, "long"=>$long ); + } + + /// set grouping attribute and function + function SetGroupBy ( $attribute, $func, $groupsort="@group desc" ) + { + assert ( is_string($attribute) ); + assert ( is_string($groupsort) ); + assert ( $func==SPH_GROUPBY_DAY + || $func==SPH_GROUPBY_WEEK + || $func==SPH_GROUPBY_MONTH + || $func==SPH_GROUPBY_YEAR + || $func==SPH_GROUPBY_ATTR + || $func==SPH_GROUPBY_ATTRPAIR ); + + $this->_groupby = $attribute; + $this->_groupfunc = $func; + $this->_groupsort = $groupsort; + } + + /// set count-distinct attribute for group-by queries + function SetGroupDistinct ( $attribute ) + { + assert ( is_string($attribute) ); + $this->_groupdistinct = $attribute; + } + + /// set distributed retries count and delay + function SetRetries ( $count, $delay=0 ) + { + assert ( is_int($count) && $count>=0 ); + assert ( is_int($delay) && $delay>=0 ); + $this->_retrycount = $count; + $this->_retrydelay = $delay; + } + + /// set result set format (hash or array; hash by default) + /// PHP specific; needed for group-by-MVA result sets that may contain duplicate IDs + function SetArrayResult ( $arrayresult ) + { + assert ( is_bool($arrayresult) ); + $this->_arrayresult = $arrayresult; + } + + /// set attribute values override + /// there can be only one override per attribute + /// $values must be a hash that maps document IDs to attribute values + function SetOverride ( $attrname, $attrtype, $values ) + { + assert ( is_string ( $attrname ) ); + assert ( in_array ( $attrtype, array ( SPH_ATTR_INTEGER, SPH_ATTR_TIMESTAMP, SPH_ATTR_BOOL, SPH_ATTR_FLOAT, SPH_ATTR_BIGINT ) ) ); + assert ( is_array ( $values ) ); + + $this->_overrides[$attrname] = array ( "attr"=>$attrname, "type"=>$attrtype, "values"=>$values ); + } + + /// set select-list (attributes or expressions), SQL-like syntax + function SetSelect ( $select ) + { + assert ( is_string ( $select ) ); + $this->_select = $select; + } + + ////////////////////////////////////////////////////////////////////////////// + + /// clear all filters (for multi-queries) + function ResetFilters () + { + $this->_filters = array(); + $this->_anchor = array(); + } + + /// clear groupby settings (for multi-queries) + function ResetGroupBy () + { + $this->_groupby = ""; + $this->_groupfunc = SPH_GROUPBY_DAY; + $this->_groupsort = "@group desc"; + $this->_groupdistinct= ""; + } + + /// clear all attribute value overrides (for multi-queries) + function ResetOverrides () + { + $this->_overrides = array (); + } + + ////////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, run given search query through given indexes, + /// and return the search results + function Query ( $query, $index="*", $comment="" ) + { + assert ( empty($this->_reqs) ); + + $this->AddQuery ( $query, $index, $comment ); + $results = $this->RunQueries (); + $this->_reqs = array (); // just in case it failed too early + + if ( !is_array($results) ) + return false; // probably network error; error message should be already filled + + $this->_error = $results[0]["error"]; + $this->_warning = $results[0]["warning"]; + if ( $results[0]["status"]==SEARCHD_ERROR ) + return false; + else + return $results[0]; + } + + /// helper to pack floats in network byte order + function _PackFloat ( $f ) + { + $t1 = pack ( "f", $f ); // machine order + list(,$t2) = unpack ( "L*", $t1 ); // int in machine order + return pack ( "N", $t2 ); + } + + /// add query to multi-query batch + /// returns index into results array from RunQueries() call + function AddQuery ( $query, $index="*", $comment="" ) + { + // mbstring workaround + $this->_MBPush (); + + // build request + $req = pack ( "NNNNN", $this->_offset, $this->_limit, $this->_mode, $this->_ranker, $this->_sort ); // mode and limits + $req .= pack ( "N", strlen($this->_sortby) ) . $this->_sortby; + $req .= pack ( "N", strlen($query) ) . $query; // query itself + $req .= pack ( "N", count($this->_weights) ); // weights + foreach ( $this->_weights as $weight ) + $req .= pack ( "N", (int)$weight ); + $req .= pack ( "N", strlen($index) ) . $index; // indexes + $req .= pack ( "N", 1 ); // id64 range marker + $req .= sphPackU64 ( $this->_min_id ) . sphPackU64 ( $this->_max_id ); // id64 range + + // filters + $req .= pack ( "N", count($this->_filters) ); + foreach ( $this->_filters as $filter ) + { + $req .= pack ( "N", strlen($filter["attr"]) ) . $filter["attr"]; + $req .= pack ( "N", $filter["type"] ); + switch ( $filter["type"] ) + { + case SPH_FILTER_VALUES: + $req .= pack ( "N", count($filter["values"]) ); + foreach ( $filter["values"] as $value ) + $req .= sphPackI64 ( $value ); + break; + + case SPH_FILTER_RANGE: + $req .= sphPackI64 ( $filter["min"] ) . sphPackI64 ( $filter["max"] ); + break; + + case SPH_FILTER_FLOATRANGE: + $req .= $this->_PackFloat ( $filter["min"] ) . $this->_PackFloat ( $filter["max"] ); + break; + + default: + assert ( 0 && "internal error: unhandled filter type" ); + } + $req .= pack ( "N", $filter["exclude"] ); + } + + // group-by clause, max-matches count, group-sort clause, cutoff count + $req .= pack ( "NN", $this->_groupfunc, strlen($this->_groupby) ) . $this->_groupby; + $req .= pack ( "N", $this->_maxmatches ); + $req .= pack ( "N", strlen($this->_groupsort) ) . $this->_groupsort; + $req .= pack ( "NNN", $this->_cutoff, $this->_retrycount, $this->_retrydelay ); + $req .= pack ( "N", strlen($this->_groupdistinct) ) . $this->_groupdistinct; + + // anchor point + if ( empty($this->_anchor) ) + { + $req .= pack ( "N", 0 ); + } else + { + $a =& $this->_anchor; + $req .= pack ( "N", 1 ); + $req .= pack ( "N", strlen($a["attrlat"]) ) . $a["attrlat"]; + $req .= pack ( "N", strlen($a["attrlong"]) ) . $a["attrlong"]; + $req .= $this->_PackFloat ( $a["lat"] ) . $this->_PackFloat ( $a["long"] ); + } + + // per-index weights + $req .= pack ( "N", count($this->_indexweights) ); + foreach ( $this->_indexweights as $idx=>$weight ) + $req .= pack ( "N", strlen($idx) ) . $idx . pack ( "N", $weight ); + + // max query time + $req .= pack ( "N", $this->_maxquerytime ); + + // per-field weights + $req .= pack ( "N", count($this->_fieldweights) ); + foreach ( $this->_fieldweights as $field=>$weight ) + $req .= pack ( "N", strlen($field) ) . $field . pack ( "N", $weight ); + + // comment + $req .= pack ( "N", strlen($comment) ) . $comment; + + // attribute overrides + $req .= pack ( "N", count($this->_overrides) ); + foreach ( $this->_overrides as $key => $entry ) + { + $req .= pack ( "N", strlen($entry["attr"]) ) . $entry["attr"]; + $req .= pack ( "NN", $entry["type"], count($entry["values"]) ); + foreach ( $entry["values"] as $id=>$val ) + { + assert ( is_numeric($id) ); + assert ( is_numeric($val) ); + + $req .= sphPackU64 ( $id ); + switch ( $entry["type"] ) + { + case SPH_ATTR_FLOAT: $req .= $this->_PackFloat ( $val ); break; + case SPH_ATTR_BIGINT: $req .= sphPackI64 ( $val ); break; + default: $req .= pack ( "N", $val ); break; + } + } + } + + // select-list + $req .= pack ( "N", strlen($this->_select) ) . $this->_select; + + // mbstring workaround + $this->_MBPop (); + + // store request to requests array + $this->_reqs[] = $req; + return count($this->_reqs)-1; + } + + /// connect to searchd, run queries batch, and return an array of result sets + function RunQueries () + { + if ( empty($this->_reqs) ) + { + $this->_error = "no queries defined, issue AddQuery() first"; + return false; + } + + // mbstring workaround + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop (); + return false; + } + + // send query, get response + $nreqs = count($this->_reqs); + $req = join ( "", $this->_reqs ); + $len = 8+strlen($req); + $req = pack ( "nnNNN", SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, $len, 0, $nreqs ) . $req; // add header + + if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_SEARCH ) ) ) + { + $this->_MBPop (); + return false; + } + + // query sent ok; we can reset reqs now + $this->_reqs = array (); + + // parse and return response + return $this->_ParseSearchResponse ( $response, $nreqs ); + } + + /// parse and return search query (or queries) response + function _ParseSearchResponse ( $response, $nreqs ) + { + $p = 0; // current position + $max = strlen($response); // max position for checks, to protect against broken responses + + $results = array (); + for ( $ires=0; $ires<$nreqs && $p<$max; $ires++ ) + { + $results[] = array(); + $result =& $results[$ires]; + + $result["error"] = ""; + $result["warning"] = ""; + + // extract status + list(,$status) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $result["status"] = $status; + if ( $status!=SEARCHD_OK ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $message = substr ( $response, $p, $len ); $p += $len; + + if ( $status==SEARCHD_WARNING ) + { + $result["warning"] = $message; + } else + { + $result["error"] = $message; + continue; + } + } + + // read schema + $fields = array (); + $attrs = array (); + + list(,$nfields) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + while ( $nfields-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $fields[] = substr ( $response, $p, $len ); $p += $len; + } + $result["fields"] = $fields; + + list(,$nattrs) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + while ( $nattrs-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attr = substr ( $response, $p, $len ); $p += $len; + list(,$type) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attrs[$attr] = $type; + } + $result["attrs"] = $attrs; + + // read match count + list(,$count) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + list(,$id64) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + + // read matches + $idx = -1; + while ( $count-->0 && $p<$max ) + { + // index into result array + $idx++; + + // parse document id and weight + if ( $id64 ) + { + $doc = sphUnpackU64 ( substr ( $response, $p, 8 ) ); $p += 8; + list(,$weight) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + } + else + { + list ( $doc, $weight ) = array_values ( unpack ( "N*N*", + substr ( $response, $p, 8 ) ) ); + $p += 8; + $doc = sphFixUint($doc); + } + $weight = sprintf ( "%u", $weight ); + + // create match entry + if ( $this->_arrayresult ) + $result["matches"][$idx] = array ( "id"=>$doc, "weight"=>$weight ); + else + $result["matches"][$doc]["weight"] = $weight; + + // parse and create attributes + $attrvals = array (); + foreach ( $attrs as $attr=>$type ) + { + // handle 64bit ints + if ( $type==SPH_ATTR_BIGINT ) + { + $attrvals[$attr] = sphUnpackI64 ( substr ( $response, $p, 8 ) ); $p += 8; + continue; + } + + // handle floats + if ( $type==SPH_ATTR_FLOAT ) + { + list(,$uval) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + list(,$fval) = unpack ( "f*", pack ( "L", $uval ) ); + $attrvals[$attr] = $fval; + continue; + } + + // handle everything else as unsigned ints + list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + if ( $type & SPH_ATTR_MULTI ) + { + $attrvals[$attr] = array (); + $nvalues = $val; + while ( $nvalues-->0 && $p<$max ) + { + list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attrvals[$attr][] = sphFixUint($val); + } + } else if ( $type==SPH_ATTR_STRING ) + { + $attrvals[$attr] = substr ( $response, $p, $val ); + $p += $val; + } else + { + $attrvals[$attr] = sphFixUint($val); + } + } + + if ( $this->_arrayresult ) + $result["matches"][$idx]["attrs"] = $attrvals; + else + $result["matches"][$doc]["attrs"] = $attrvals; + } + + list ( $total, $total_found, $msecs, $words ) = + array_values ( unpack ( "N*N*N*N*", substr ( $response, $p, 16 ) ) ); + $result["total"] = sprintf ( "%u", $total ); + $result["total_found"] = sprintf ( "%u", $total_found ); + $result["time"] = sprintf ( "%.3f", $msecs/1000 ); + $p += 16; + + while ( $words-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $word = substr ( $response, $p, $len ); $p += $len; + list ( $docs, $hits ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; + $result["words"][$word] = array ( + "docs"=>sprintf ( "%u", $docs ), + "hits"=>sprintf ( "%u", $hits ) ); + } + } + + $this->_MBPop (); + return $results; + } + + ///////////////////////////////////////////////////////////////////////////// + // excerpts generation + ///////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, and generate exceprts (snippets) + /// of given documents for given query. returns false on failure, + /// an array of snippets on success + function BuildExcerpts ( $docs, $index, $words, $opts=array() ) + { + assert ( is_array($docs) ); + assert ( is_string($index) ); + assert ( is_string($words) ); + assert ( is_array($opts) ); + + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return false; + } + + ///////////////// + // fixup options + ///////////////// + + if ( !isset($opts["before_match"]) ) $opts["before_match"] = ""; + if ( !isset($opts["after_match"]) ) $opts["after_match"] = ""; + if ( !isset($opts["chunk_separator"]) ) $opts["chunk_separator"] = " ... "; + if ( !isset($opts["limit"]) ) $opts["limit"] = 256; + if ( !isset($opts["limit_passages"]) ) $opts["limit_passages"] = 0; + if ( !isset($opts["limit_words"]) ) $opts["limit_words"] = 0; + if ( !isset($opts["around"]) ) $opts["around"] = 5; + if ( !isset($opts["exact_phrase"]) ) $opts["exact_phrase"] = false; + if ( !isset($opts["single_passage"]) ) $opts["single_passage"] = false; + if ( !isset($opts["use_boundaries"]) ) $opts["use_boundaries"] = false; + if ( !isset($opts["weight_order"]) ) $opts["weight_order"] = false; + if ( !isset($opts["query_mode"]) ) $opts["query_mode"] = false; + if ( !isset($opts["force_all_words"]) ) $opts["force_all_words"] = false; + if ( !isset($opts["start_passage_id"]) ) $opts["start_passage_id"] = 1; + if ( !isset($opts["load_files"]) ) $opts["load_files"] = false; + if ( !isset($opts["html_strip_mode"]) ) $opts["html_strip_mode"] = "index"; + if ( !isset($opts["allow_empty"]) ) $opts["allow_empty"] = false; + if ( !isset($opts["passage_boundary"]) ) $opts["passage_boundary"] = "none"; + if ( !isset($opts["emit_zones"]) ) $opts["emit_zones"] = false; + + ///////////////// + // build request + ///////////////// + + // v.1.2 req + $flags = 1; // remove spaces + if ( $opts["exact_phrase"] ) $flags |= 2; + if ( $opts["single_passage"] ) $flags |= 4; + if ( $opts["use_boundaries"] ) $flags |= 8; + if ( $opts["weight_order"] ) $flags |= 16; + if ( $opts["query_mode"] ) $flags |= 32; + if ( $opts["force_all_words"] ) $flags |= 64; + if ( $opts["load_files"] ) $flags |= 128; + if ( $opts["allow_empty"] ) $flags |= 256; + if ( $opts["emit_zones"] ) $flags |= 512; + $req = pack ( "NN", 0, $flags ); // mode=0, flags=$flags + $req .= pack ( "N", strlen($index) ) . $index; // req index + $req .= pack ( "N", strlen($words) ) . $words; // req words + + // options + $req .= pack ( "N", strlen($opts["before_match"]) ) . $opts["before_match"]; + $req .= pack ( "N", strlen($opts["after_match"]) ) . $opts["after_match"]; + $req .= pack ( "N", strlen($opts["chunk_separator"]) ) . $opts["chunk_separator"]; + $req .= pack ( "NN", (int)$opts["limit"], (int)$opts["around"] ); + $req .= pack ( "NNN", (int)$opts["limit_passages"], (int)$opts["limit_words"], (int)$opts["start_passage_id"] ); // v.1.2 + $req .= pack ( "N", strlen($opts["html_strip_mode"]) ) . $opts["html_strip_mode"]; + $req .= pack ( "N", strlen($opts["passage_boundary"]) ) . $opts["passage_boundary"]; + + // documents + $req .= pack ( "N", count($docs) ); + foreach ( $docs as $doc ) + { + assert ( is_string($doc) ); + $req .= pack ( "N", strlen($doc) ) . $doc; + } + + //////////////////////////// + // send query, get response + //////////////////////////// + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, $len ) . $req; // add header + if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_EXCERPT ) ) ) + { + $this->_MBPop (); + return false; + } + + ////////////////// + // parse response + ////////////////// + + $pos = 0; + $res = array (); + $rlen = strlen($response); + for ( $i=0; $i $rlen ) + { + $this->_error = "incomplete reply"; + $this->_MBPop (); + return false; + } + $res[] = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + } + + $this->_MBPop (); + return $res; + } + + + ///////////////////////////////////////////////////////////////////////////// + // keyword generation + ///////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, and generate keyword list for a given query + /// returns false on failure, + /// an array of words on success + function BuildKeywords ( $query, $index, $hits ) + { + assert ( is_string($query) ); + assert ( is_string($index) ); + assert ( is_bool($hits) ); + + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return false; + } + + ///////////////// + // build request + ///////////////// + + // v.1.0 req + $req = pack ( "N", strlen($query) ) . $query; // req query + $req .= pack ( "N", strlen($index) ) . $index; // req index + $req .= pack ( "N", (int)$hits ); + + //////////////////////////// + // send query, get response + //////////////////////////// + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, $len ) . $req; // add header + if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_KEYWORDS ) ) ) + { + $this->_MBPop (); + return false; + } + + ////////////////// + // parse response + ////////////////// + + $pos = 0; + $res = array (); + $rlen = strlen($response); + list(,$nwords) = unpack ( "N*", substr ( $response, $pos, 4 ) ); + $pos += 4; + for ( $i=0; $i<$nwords; $i++ ) + { + list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; + $tokenized = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + + list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; + $normalized = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + + $res[] = array ( "tokenized"=>$tokenized, "normalized"=>$normalized ); + + if ( $hits ) + { + list($ndocs,$nhits) = array_values ( unpack ( "N*N*", substr ( $response, $pos, 8 ) ) ); + $pos += 8; + $res [$i]["docs"] = $ndocs; + $res [$i]["hits"] = $nhits; + } + + if ( $pos > $rlen ) + { + $this->_error = "incomplete reply"; + $this->_MBPop (); + return false; + } + } + + $this->_MBPop (); + return $res; + } + + function EscapeString ( $string ) + { + $from = array ( '\\', '(',')','|','-','!','@','~','"','&', '/', '^', '$', '=' ); + $to = array ( '\\\\', '\(','\)','\|','\-','\!','\@','\~','\"', '\&', '\/', '\^', '\$', '\=' ); + + return str_replace ( $from, $to, $string ); + } + + ///////////////////////////////////////////////////////////////////////////// + // attribute updates + ///////////////////////////////////////////////////////////////////////////// + + /// batch update given attributes in given rows in given indexes + /// returns amount of updated documents (0 or more) on success, or -1 on failure + function UpdateAttributes ( $index, $attrs, $values, $mva=false ) + { + // verify everything + assert ( is_string($index) ); + assert ( is_bool($mva) ); + + assert ( is_array($attrs) ); + foreach ( $attrs as $attr ) + assert ( is_string($attr) ); + + assert ( is_array($values) ); + foreach ( $values as $id=>$entry ) + { + assert ( is_numeric($id) ); + assert ( is_array($entry) ); + assert ( count($entry)==count($attrs) ); + foreach ( $entry as $v ) + { + if ( $mva ) + { + assert ( is_array($v) ); + foreach ( $v as $vv ) + assert ( is_int($vv) ); + } else + assert ( is_int($v) ); + } + } + + // build request + $this->_MBPush (); + $req = pack ( "N", strlen($index) ) . $index; + + $req .= pack ( "N", count($attrs) ); + foreach ( $attrs as $attr ) + { + $req .= pack ( "N", strlen($attr) ) . $attr; + $req .= pack ( "N", $mva ? 1 : 0 ); + } + + $req .= pack ( "N", count($values) ); + foreach ( $values as $id=>$entry ) + { + $req .= sphPackU64 ( $id ); + foreach ( $entry as $v ) + { + $req .= pack ( "N", $mva ? count($v) : $v ); + if ( $mva ) + foreach ( $v as $vv ) + $req .= pack ( "N", $vv ); + } + } + + // connect, send query, get response + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop (); + return -1; + } + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, $len ) . $req; // add header + if ( !$this->_Send ( $fp, $req, $len+8 ) ) + { + $this->_MBPop (); + return -1; + } + + if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_UPDATE ) )) + { + $this->_MBPop (); + return -1; + } + + // parse response + list(,$updated) = unpack ( "N*", substr ( $response, 0, 4 ) ); + $this->_MBPop (); + return $updated; + } + + ///////////////////////////////////////////////////////////////////////////// + // persistent connections + ///////////////////////////////////////////////////////////////////////////// + + function Open() + { + if ( $this->_socket !== false ) + { + $this->_error = 'already connected'; + return false; + } + if ( !$fp = $this->_Connect() ) + return false; + + // command, command version = 0, body length = 4, body = 1 + $req = pack ( "nnNN", SEARCHD_COMMAND_PERSIST, 0, 4, 1 ); + if ( !$this->_Send ( $fp, $req, 12 ) ) + return false; + + $this->_socket = $fp; + return true; + } + + function Close() + { + if ( $this->_socket === false ) + { + $this->_error = 'not connected'; + return false; + } + + fclose ( $this->_socket ); + $this->_socket = false; + + return true; + } + + ////////////////////////////////////////////////////////////////////////// + // status + ////////////////////////////////////////////////////////////////////////// + + function Status () + { + $this->_MBPush (); + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return false; + } + + $req = pack ( "nnNN", SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, 1 ); // len=4, body=1 + if ( !( $this->_Send ( $fp, $req, 12 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_STATUS ) ) ) + { + $this->_MBPop (); + return false; + } + + $res = substr ( $response, 4 ); // just ignore length, error handling, etc + $p = 0; + list ( $rows, $cols ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; + + $res = array(); + for ( $i=0; $i<$rows; $i++ ) + for ( $j=0; $j<$cols; $j++ ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $res[$i][] = substr ( $response, $p, $len ); $p += $len; + } + + $this->_MBPop (); + return $res; + } + + ////////////////////////////////////////////////////////////////////////// + // flush + ////////////////////////////////////////////////////////////////////////// + + function FlushAttributes () + { + $this->_MBPush (); + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return -1; + } + + $req = pack ( "nnN", SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0 ); // len=0 + if ( !( $this->_Send ( $fp, $req, 8 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_FLUSHATTRS ) ) ) + { + $this->_MBPop (); + return -1; + } + + $tag = -1; + if ( strlen($response)==4 ) + list(,$tag) = unpack ( "N*", $response ); + else + $this->_error = "unexpected response length"; + + $this->_MBPop (); + return $tag; + } +} + +// +// $Id: sphinxapi.php 2758 2011-04-04 11:10:44Z kevg $ +// -- cgit v1.2.3-54-g00ecf From 9a6456fa121db77759f5a2cde0f7d7df51cd2bc9 Mon Sep 17 00:00:00 2001 From: Rob Hoelz Date: Tue, 22 Apr 2014 22:28:32 -0500 Subject: Stop after first search plugin The sanity check should have caught any extra ones, but it doesn't hurt to be careful --- include/functions2.php | 1 + 1 file changed, 1 insertion(+) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 22c602362..959e7154d 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -399,6 +399,7 @@ if ($search) { foreach (PluginHost::getInstance()->get_hooks(PluginHost::HOOK_SEARCH) as $plugin) { list($search_query_part, $search_words) = $plugin->hook_search($search); + break; } // fall back in case of no plugins -- cgit v1.2.3-54-g00ecf From 2bb11658a8b032dc345b402de25265befa54608e Mon Sep 17 00:00:00 2001 From: Dave Zaikos Date: Mon, 12 May 2014 00:59:27 -0400 Subject: Added HOOK_FORMAT_ENCLOSURES plugin hook. Runs HTML and enclosures array through a plugin hook when rendering an article's enclosures in format_article_enclosures(). Allows plugins to override handling of how enclosures are presented by either filtering the array of enclosures, or generating the HTML to add to the article content. --- classes/pluginhost.php | 1 + include/functions2.php | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'include/functions2.php') diff --git a/classes/pluginhost.php b/classes/pluginhost.php index 4582c314a..bd6afa81d 100644 --- a/classes/pluginhost.php +++ b/classes/pluginhost.php @@ -40,6 +40,7 @@ class PluginHost { const HOOK_QUERY_HEADLINES = 23; const HOOK_HOUSE_KEEPING = 24; const HOOK_SEARCH = 25; + const HOOK_FORMAT_ENCLOSURES = 26; const KIND_ALL = 1; const KIND_SYSTEM = 2; diff --git a/include/functions2.php b/include/functions2.php index 959e7154d..cf1fba17c 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -1846,8 +1846,17 @@ $result = get_article_enclosures($id); $rv = ''; - if (count($result) > 0) { + foreach (PluginHost::getInstance()->get_hooks(PluginHost::HOOK_FORMAT_ENCLOSURES) as $plugin) { + $retval = $plugin->hook_format_enclosures($rv, $result, $id, $always_display_enclosures, $article_content, $hide_images); + if (is_array($retval)) { + $rv = $retval[0]; + $result = $retval[1]; + } else { + $rv = $retval; + } + } + if ($rv === '' && is_array($result)) { $entries_html = array(); $entries = array(); $entries_inline = array(); -- cgit v1.2.3-54-g00ecf From c6ce584dd56320da336ecbf69d424f0bc05f4f5a Mon Sep 17 00:00:00 2001 From: Dave Zaikos Date: Mon, 12 May 2014 13:54:04 -0400 Subject: Changed how the enclosures array is checked for content so an empty array is not processed. Fixes a change from the previous branch commit. --- include/functions2.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index cf1fba17c..5228d875c 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -1856,7 +1856,7 @@ } } - if ($rv === '' && is_array($result)) { + if ($rv === '' && !empty($result)) { $entries_html = array(); $entries = array(); $entries_inline = array(); -- cgit v1.2.3-54-g00ecf From 7b3110c9c97346c7625fba81238d2119381e2247 Mon Sep 17 00:00:00 2001 From: Bram Schoenmakers Date: Thu, 19 Jun 2014 10:29:48 +0200 Subject: Fix for testing filters getting stuck sometimes. Sometimes when testing a filter the database becomes unresponsive, stuck in a long query. Solution: properly join ttrss_entries and ttrss_user_entries instead of working with their carthesian product. --- include/functions2.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 5228d875c..507c5343e 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -426,8 +426,10 @@ // Try to check if SQL regexp implementation chokes on a valid regexp - $result = db_query("SELECT true AS true_val FROM ttrss_entries, - ttrss_user_entries, ttrss_feeds + $result = db_query("SELECT true AS true_val + FROM ttrss_entries + JOIN ttrss_user_entries ON ttrss_entries.id = ttrss_user_entries.ref_id + JOIN ttrss_feeds ON ttrss_feeds.id = ttrss_user_entries.feed_id WHERE $filter_query_part LIMIT 1", false); if ($result) { -- cgit v1.2.3-54-g00ecf From 3e0f2090bfe90756b6a0a8f803ec5c407b50c41e Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 2 Jul 2014 10:38:59 +0400 Subject: mail plugin: cleanup disabled completion, add configurable destination email list --- include/functions2.php | 4 +-- plugins/mail/init.php | 68 ++++++++++++++++++++++++++++++++++++++++++++------ plugins/mail/mail.js | 4 +-- 3 files changed, 64 insertions(+), 12 deletions(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 507c5343e..d6a399ffc 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -1747,7 +1747,7 @@ return $rv; } - function save_email_address($email) { + /* function save_email_address($email) { // FIXME: implement persistent storage of emails if (!$_SESSION['stored_emails']) @@ -1755,7 +1755,7 @@ if (!in_array($email, $_SESSION['stored_emails'])) array_push($_SESSION['stored_emails'], $email); - } + } */ function get_feed_access_key($feed_id, $is_cat, $owner_uid = false) { diff --git a/plugins/mail/init.php b/plugins/mail/init.php index 5fcdf32af..b2d73d4eb 100644 --- a/plugins/mail/init.php +++ b/plugins/mail/init.php @@ -13,12 +13,61 @@ class Mail extends Plugin { $this->host = $host; $host->add_hook($host::HOOK_ARTICLE_BUTTON, $this); + $host->add_hook($host::HOOK_PREFS_TAB, $this); } function get_js() { return file_get_contents(dirname(__FILE__) . "/mail.js"); } + function save() { + $addresslist = db_escape_string($_POST["addresslist"]); + + $this->host->set($this, "addresslist", $addresslist); + + echo __("Mail addresses saved."); + } + + function hook_prefs_tab($args) { + if ($args != "prefPrefs") return; + + print "
"; + + print "

" . __("You can set predefined email addressed here (comma-separated list):") . "

"; + + print "
"; + + print ""; + + print ""; + print ""; + print ""; + + $addresslist = $this->host->get($this, "addresslist"); + + print ""; + + print "

"; + + print "

"; + + print "
"; + } + function hook_article_button($line) { return "readTemplateFromFile("templates/email_article_template.txt"); @@ -96,16 +144,20 @@ class Mail extends Plugin { print ""; + $addresslist = explode(",", $this->host->get($this, "addresslist")); + print __('To:'); print ""; - print ""; + name=\"destination\" id=\"emailArticleDlg_destination\">"; */ - print "
"; + print_select("destination", "", $addresslist, 'dojoType="dijit.form.FilteringSelect"'); + +/* print "
"; */ print ""; @@ -155,14 +207,14 @@ class Mail extends Plugin { if (!$rc) { $reply['error'] = $mail->ErrorInfo; } else { - save_email_address(db_escape_string($destination)); + //save_email_address(db_escape_string($destination)); $reply['message'] = "UPDATE_COUNTERS"; } print json_encode($reply); } - function completeEmails() { + /* function completeEmails() { $search = db_escape_string($_REQUEST["search"]); print "
    "; @@ -174,7 +226,7 @@ class Mail extends Plugin { } print "
"; - } + } */ function api_version() { return 2; diff --git a/plugins/mail/mail.js b/plugins/mail/mail.js index 6166f01c1..db0503ff9 100644 --- a/plugins/mail/mail.js +++ b/plugins/mail/mail.js @@ -44,13 +44,13 @@ function emailArticle(id) { }, href: query}); - var tmph = dojo.connect(dialog, 'onLoad', function() { + /* var tmph = dojo.connect(dialog, 'onLoad', function() { dojo.disconnect(tmph); new Ajax.Autocompleter('emailArticleDlg_destination', 'emailArticleDlg_dst_choices', "backend.php?op=pluginhandler&plugin=mail&method=completeEmails", { tokens: '', paramName: "search" }); - }); + }); */ dialog.show(); -- cgit v1.2.3-54-g00ecf From 0c019b4436c9ec224b23972a65aaf55aa03d5d57 Mon Sep 17 00:00:00 2001 From: Dave Zaikos Date: Sun, 22 Jun 2014 01:08:57 -0400 Subject: Added curl CURLOPT_NOBODY option in geturl function to ensure the HEAD method is used when resolving permanent/temporary URL moves. --- include/functions2.php | 1 + 1 file changed, 1 insertion(+) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index d6a399ffc..3e3367240 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -2220,6 +2220,7 @@ curl_setopt($curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0 Firefox/5.0'); curl_setopt($curl, CURLOPT_HTTPHEADER, $header); curl_setopt($curl, CURLOPT_HEADER, true); + curl_setopt($curl, CURLOPT_NOBODY, true); curl_setopt($curl, CURLOPT_REFERER, $url); curl_setopt($curl, CURLOPT_ENCODING, 'gzip,deflate'); curl_setopt($curl, CURLOPT_AUTOREFERER, true); -- cgit v1.2.3-54-g00ecf From 1e871938153497d46537fc313a79155717d532d0 Mon Sep 17 00:00:00 2001 From: Felix Eckhofer Date: Tue, 15 Jul 2014 15:44:05 +0200 Subject: Add width/height attribute to image enclosures --- include/functions2.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 3e3367240..07024d38f 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -1868,6 +1868,8 @@ $url = $line["content_url"]; $ctype = $line["content_type"]; $title = $line["title"]; + $width = $line["width"]; + $height = $line["height"]; if (!$ctype) $ctype = __("unknown type"); @@ -1891,6 +1893,8 @@ $entry["filename"] = $filename; $entry["url"] = $url; $entry["title"] = $title; + $entry["width"] = $width; + $entry["height"] = $height; array_push($entries, $entry); } @@ -1905,9 +1909,15 @@ preg_match("/\.(jpg|png|gif|bmp)/i", $entry["filename"])) { if (!$hide_images) { + $encsize = ''; + if ($entry['height'] > 0) + $encsize .= ' height="' . intval($entry['width']) . '"'; + if ($entry['width'] > 0) + $encsize .= ' width="' . intval($entry['height']) . '"'; $rv .= "

\"".htmlspecialchars($entry["filename"])."\"

"; + src=\"" .htmlspecialchars($entry["url"]) . "\" + " . $encsize . " />

"; } else { $rv .= "

Date: Tue, 19 Aug 2014 14:50:25 +0400 Subject: get_minified_js: store and check tt-rss version in cached files --- include/functions2.php | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 07024d38f..9d8bc76aa 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -2287,19 +2287,26 @@ if (!isset($_GET['debug'])) { $cached_file = CACHE_DIR . "/js/".basename($js).".js"; - if (file_exists($cached_file) && - is_readable($cached_file) && - filemtime($cached_file) >= filemtime("js/$js.js")) { + if (file_exists($cached_file) && is_readable($cached_file) && filemtime($cached_file) >= filemtime("js/$js.js")) { - $rv .= file_get_contents($cached_file); + list($header, $contents) = explode("\n", file_get_contents($cached_file), 2); - } else { - $minified = JShrink\Minifier::minify(file_get_contents("js/$js.js")); - file_put_contents($cached_file, $minified); - $rv .= $minified; + if ($header && $contents) { + list($htag, $hversion) = explode(":", $header); + + if ($htag == "tt-rss" && $hversion == VERSION) { + $rv .= $contents; + continue; + } + } } + + $minified = JShrink\Minifier::minify(file_get_contents("js/$js.js")); + file_put_contents($cached_file, "tt-rss:" . VERSION . "\n" . $minified); + $rv .= $minified; + } else { - $rv .= file_get_contents("js/$js.js"); + $rv .= file_get_contents("js/$js.js"); // no cache in debug mode } } -- cgit v1.2.3-54-g00ecf From fafac207c5a7bb610c4a2ce473cc677eeca92e4e Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 20 Aug 2014 12:01:41 +0400 Subject: geturl: if head request is denied because host is still living in 20th century, try requesting body (thanks to incompetent admins of arxiv.org) --- include/functions.php | 2 +- include/functions2.php | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'include/functions2.php') diff --git a/include/functions.php b/include/functions.php index c595944e1..ac8fb6302 100644 --- a/include/functions.php +++ b/include/functions.php @@ -360,7 +360,7 @@ $fetch_curl_used = true; - if (ini_get("safe_mode") || ini_get("open_basedir")) { + if (ini_get("safe_mode") || ini_get("open_basedir") || defined("FORCE_GETURL")) { $new_url = geturl($url); if (!$new_url) { // geturl has already populated $fetch_last_error diff --git a/include/functions2.php b/include/functions2.php index 9d8bc76aa..555aa04e9 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -2209,7 +2209,7 @@ return in_array($interface, class_implements($class)); } - function geturl($url, $depth = 0){ + function geturl($url, $depth = 0, $nobody = true){ if ($depth == 20) return $url; @@ -2230,7 +2230,7 @@ curl_setopt($curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0 Firefox/5.0'); curl_setopt($curl, CURLOPT_HTTPHEADER, $header); curl_setopt($curl, CURLOPT_HEADER, true); - curl_setopt($curl, CURLOPT_NOBODY, true); + curl_setopt($curl, CURLOPT_NOBODY, $nobody); curl_setopt($curl, CURLOPT_REFERER, $url); curl_setopt($curl, CURLOPT_ENCODING, 'gzip,deflate'); curl_setopt($curl, CURLOPT_AUTOREFERER, true); @@ -2252,6 +2252,13 @@ $status = curl_getinfo($curl); if($status['http_code']!=200){ + + // idiot site not allowing http get + if($status['http_code'] == 405) { + curl_close($curl); + return geturl($url, $depth +1, false); + } + if($status['http_code'] == 301 || $status['http_code'] == 302) { curl_close($curl); list($header) = explode("\r\n\r\n", $html, 2); -- cgit v1.2.3-54-g00ecf From ae962a96fdba3fcc14b60d48f0db48bb7f3b243e Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 20 Aug 2014 12:04:22 +0400 Subject: fix typo --- include/functions2.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/functions2.php') diff --git a/include/functions2.php b/include/functions2.php index 555aa04e9..deeaaf210 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -2253,7 +2253,7 @@ if($status['http_code']!=200){ - // idiot site not allowing http get + // idiot site not allowing http head if($status['http_code'] == 405) { curl_close($curl); return geturl($url, $depth +1, false); -- cgit v1.2.3-54-g00ecf From 8ef0ca2fa66ba9110d410dc85c6ed84c85d58cc9 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 20 Aug 2014 12:31:10 +0400 Subject: remove unused format_libxml_error --- include/functions.php | 12 ------------ include/functions2.php | 5 ----- 2 files changed, 17 deletions(-) (limited to 'include/functions2.php') diff --git a/include/functions.php b/include/functions.php index ac8fb6302..764dea10e 100644 --- a/include/functions.php +++ b/include/functions.php @@ -1710,18 +1710,6 @@ $url = key($feedUrls); } - /* libxml_use_internal_errors(true); - $doc = new DOMDocument(); - $doc->loadXML($contents); - $error = libxml_get_last_error(); - libxml_clear_errors(); - - if ($error) { - $error_message = format_libxml_error($error); - - return array("code" => 6, "message" => $error_message); - } */ - if ($cat_id == "0" || !$cat_id) { $cat_qpart = "NULL"; } else { diff --git a/include/functions2.php b/include/functions2.php index deeaaf210..69f447e8c 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -2404,9 +2404,4 @@ return LABEL_BASE_INDEX - 1 + abs($feed); } - function format_libxml_error($error) { - return T_sprintf("LibXML error %s at line %d (column %d): %s", - $error->code, $error->line, $error->column, - $error->message); - } ?> -- cgit v1.2.3-54-g00ecf From 229a871be7eec8e96c9fa0efab50e40ccc6c4ef0 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Thu, 16 Oct 2014 09:01:23 +0400 Subject: do not force sslversion in any cases --- include/functions.php | 4 ---- include/functions2.php | 4 ---- 2 files changed, 8 deletions(-) (limited to 'include/functions2.php') diff --git a/include/functions.php b/include/functions.php index d783bd853..1dbf004da 100644 --- a/include/functions.php +++ b/include/functions.php @@ -403,10 +403,6 @@ curl_setopt($ch, CURLOPT_POSTFIELDS, $post_query); } - if ((OPENSSL_VERSION_NUMBER >= 0x0090808f) && (OPENSSL_VERSION_NUMBER < 0x10000000)) { - curl_setopt($ch, CURLOPT_SSLVERSION, 3); - } - if ($login && $pass) curl_setopt($ch, CURLOPT_USERPWD, "$login:$pass"); diff --git a/include/functions2.php b/include/functions2.php index 69f447e8c..672373e6d 100644 --- a/include/functions2.php +++ b/include/functions2.php @@ -2243,10 +2243,6 @@ curl_setopt($curl, CURLOPT_PROXY, _CURL_HTTP_PROXY); } - if ((OPENSSL_VERSION_NUMBER >= 0x0090808f) && (OPENSSL_VERSION_NUMBER < 0x10000000)) { - curl_setopt($curl, CURLOPT_SSLVERSION, 3); - } - $html = curl_exec($curl); $status = curl_getinfo($curl); -- cgit v1.2.3-54-g00ecf