From 0fd041a35372a0b5aed12be2be2097bddb66982b Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 16:23:53 +0100 Subject: [PATCH 1/9] Add timeouts for failing instances --- config.php.example | 3 +++ engines/librex/fallback.php | 37 ++++++++++++++++++++++++++----------- misc/cooldowns.php | 20 ++++++++++++++++++++ misc/search_engine.php | 1 + 4 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 misc/cooldowns.php diff --git a/config.php.example b/config.php.example index f906c724..599278b6 100644 --- a/config.php.example +++ b/config.php.example @@ -20,6 +20,9 @@ // This may greatly increase the time it takes to get a result and in some cases results in 504 errors "instance_fallback" => false, + // how long in minutes to put google/other instances on cooldown if they aren't responding + "request_cooldown" => 5, + /* Preset privacy friendly frontends for users, these can be overwritten by users in the settings e.g.: Preset the invidious instance URL: "instance_url" => "https://yewtu.be", diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 7d5e6a10..5244c385 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -15,25 +15,38 @@ public function get_results() { $response = json_decode($response, true); if (!$response) return array(); - + return array_values($response); } } + function load_instances($cooldowns) { + $instances_json = json_decode(file_get_contents("instances.json"), true); + + if (empty($instances_json["instances"])) + return array(); + + $instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet']))); + $instances = array_filter($instances, fn($n) => !has_cooldown($n, $cooldowns)); + shuffle($instances); + return $instances; + } function get_librex_results($opts) { if (!$opts->do_fallback) return array(); - $instances_json = json_decode(file_get_contents("instances.json"), true); + require "misc/cooldowns.php"; - if (empty($instances_json["instances"])) - return array(); + $cooldowns = load_cooldowns(); + error_log("loaded" . count($cooldowns)); + error_log(print_r($cooldowns,true)); - // TODO pick instances which aren't on cooldown + echo "
";
+        print_r($cooldowns);
+        echo "
"; - $instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet']))); - shuffle($instances); + $instances = load_instances($cooldowns); $results = array(); $tries = 0; @@ -43,6 +56,7 @@ function get_librex_results($opts) { $instance = array_pop($instances); + error_log($instance . "--- " . parse_url($instance)["host"]); if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"]) continue; @@ -52,12 +66,13 @@ function get_librex_results($opts) { if (count($results) > 1) return $results; - } while ( !empty($instances)); + // on fail then do this + $timeout = ($opts->request_cooldown ?? "1") * 60; + $cooldowns = set_cooldown($instance, $timeout, $cooldowns); - if (empty($instances)) - return array(); + } while (!empty($instances)); - return array_values($results); + return array(); } ?> diff --git a/misc/cooldowns.php b/misc/cooldowns.php new file mode 100644 index 00000000..975afc02 --- /dev/null +++ b/misc/cooldowns.php @@ -0,0 +1,20 @@ + time(); + } +?> diff --git a/misc/search_engine.php b/misc/search_engine.php index 1c1c1002..0b2e2217 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -57,6 +57,7 @@ function load_opts() { foreach (array_keys($opts->frontends ?? array()) as $frontend) { $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"]; } + return $opts; } From f50022f7807683aaaa4e72dd78123e575baf6a6c Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 17:39:08 +0100 Subject: [PATCH 2/9] added request cooldowns for google --- config.php.example | 3 ++- engines/librex/fallback.php | 4 +--- engines/text/google.php | 3 +-- engines/text/text.php | 17 ++++++++++++++--- misc/search_engine.php | 3 +++ 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/config.php.example b/config.php.example index 599278b6..b69c1dca 100644 --- a/config.php.example +++ b/config.php.example @@ -151,7 +151,8 @@ CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, CURLOPT_MAXREDIRS => 5, CURLOPT_TIMEOUT => 3, - CURLOPT_VERBOSE => false + CURLOPT_VERBOSE => false, + CURLOPT_FOLLOWLOCATION => true ) ); ?> diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 5244c385..a56c7bbc 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -36,9 +36,7 @@ function get_librex_results($opts) { if (!$opts->do_fallback) return array(); - require "misc/cooldowns.php"; - - $cooldowns = load_cooldowns(); + $cooldowns = $opts->cooldowns; error_log("loaded" . count($cooldowns)); error_log(print_r($cooldowns,true)); diff --git a/engines/text/google.php b/engines/text/google.php index d43ff1d2..45c871d8 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -1,7 +1,6 @@ query)); $results = array(); diff --git a/engines/text/text.php b/engines/text/text.php index ea983beb..2122d65a 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -5,19 +5,23 @@ public function __construct($opts, $mh) { $this->page = $opts->page; $this->opts = $opts; - $engine = $opts->preferred_engines["text"] ?? "google"; + $this->engine = $opts->preferred_engines["text"] ?? "google"; $query_parts = explode(" ", $this->query); $last_word_query = end($query_parts); if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") check_ddg_bang($this->query, $opts); - if ($engine == "google") { + if (has_cooldown($this->engine, $this->opts->cooldowns)) + return; + + if ($this->engine == "google") { + require "engines/text/google.php"; $this->engine_request = new GoogleRequest($opts, $mh); } - if ($engine == "duckduckgo") { + if ($this->engine == "duckduckgo") { require "engines/text/duckduckgo.php"; $this->engine_request = new DuckDuckGoRequest($opts, $mh); } @@ -27,6 +31,10 @@ public function __construct($opts, $mh) { } public function get_results() { + if (!$this->engine_request) + return array(); + + error_log("fetching googl results"); $results = $this->engine_request->get_results(); if ($this->special_request) { @@ -36,6 +44,9 @@ public function get_results() { $results = array_merge(array($special_result), $results); } + if (count($results) <= 1) + set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns); + return $results; } diff --git a/misc/search_engine.php b/misc/search_engine.php index 0b2e2217..89f91763 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -111,6 +111,9 @@ function init_search($opts, $mh) { } function fetch_search_results($opts, $do_print) { + require "misc/cooldowns.php"; + $opts->cooldowns = load_cooldowns(); + $start_time = microtime(true); $mh = curl_multi_init(); $search_category = init_search($opts, $mh); From 039e549d4ce11143521b7909bcb563b9ea178cc3 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 17:48:45 +0100 Subject: [PATCH 3/9] Change default timeout to 25 minutes --- config.php.example | 6 +++--- engines/librex/fallback.php | 8 -------- engines/text/text.php | 1 - misc/cooldowns.php | 1 - 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/config.php.example b/config.php.example index b69c1dca..567e839e 100644 --- a/config.php.example +++ b/config.php.example @@ -17,11 +17,11 @@ "disable_hidden_service_search" => false, // Fallback to another librex instance if google search fails - // This may greatly increase the time it takes to get a result and in some cases results in 504 errors - "instance_fallback" => false, + // This may greatly increase the time it takes to get a result, if a direct search is not possible + "instance_fallback" => true, // how long in minutes to put google/other instances on cooldown if they aren't responding - "request_cooldown" => 5, + "request_cooldown" => 25, /* Preset privacy friendly frontends for users, these can be overwritten by users in the settings diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index a56c7bbc..4ff13b00 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -37,13 +37,6 @@ function get_librex_results($opts) { return array(); $cooldowns = $opts->cooldowns; - error_log("loaded" . count($cooldowns)); - error_log(print_r($cooldowns,true)); - - echo "
";
-        print_r($cooldowns);
-        echo "
"; - $instances = load_instances($cooldowns); $results = array(); @@ -54,7 +47,6 @@ function get_librex_results($opts) { $instance = array_pop($instances); - error_log($instance . "--- " . parse_url($instance)["host"]); if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"]) continue; diff --git a/engines/text/text.php b/engines/text/text.php index 2122d65a..26af3ae4 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -34,7 +34,6 @@ public function get_results() { if (!$this->engine_request) return array(); - error_log("fetching googl results"); $results = $this->engine_request->get_results(); if ($this->special_request) { diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 975afc02..70d0bbe9 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -9,7 +9,6 @@ function save_cooldowns($cooldowns) { function set_cooldown($instance, $timeout, $cooldowns) { $cooldowns[$instance] = time() + $timeout; - error_log("cooldown on instnace $instance for $timeout seconds"); save_cooldowns($cooldowns); return $cooldowns; } From b8047b298247d061d8e34bc58d2d5b1373f98936 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 17:53:59 +0100 Subject: [PATCH 4/9] add default for request_cooldown --- misc/search_engine.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/search_engine.php b/misc/search_engine.php index 89f91763..75e87a5d 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -33,6 +33,8 @@ static public function print_results($results){} function load_opts() { $opts = require "config.php"; + $opts->request_cooldown ??= 25; + $opts->query = trim($_REQUEST["q"] ?? ""); $opts->type = (int) ($_REQUEST["t"] ?? 0); $opts->page = (int) ($_REQUEST["p"] ?? 0); From 8bae58c2b28dcc3f9d1eec2291e09a4a76070b85 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 18:06:12 +0100 Subject: [PATCH 5/9] Add php8-apcu to dependencies --- README.md | 2 +- docker/php/php.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f56505af..44e781ad 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ You can access the full list of LibreX and LibreY instances on one of the follow ### About LibreY LibreY gives you text results from DuckDuckGo or Google, images from Qwant, and torrents from i.e. Ahmia and popular torrent sites without spying on you. -
LibreY doesn't save any type of data about the user, there are no logs (except NGINX logs if the host sets them), no caches. +
LibreY doesn't save **any** type of data about the user, there are no logs (except NGINX logs if the host sets them). ### LibreY compared to other metasearch engines diff --git a/docker/php/php.dockerfile b/docker/php/php.dockerfile index 5a522658..87351377 100644 --- a/docker/php/php.dockerfile +++ b/docker/php/php.dockerfile @@ -56,7 +56,7 @@ ENV CURLOPT_VERBOSE=true # Install PHP-FPM using Alpine's package manager, apk # Configure PHP-FPM to listen on a Unix socket instead of a TCP port, which is more secure and efficient -RUN apk add php8 php8-fpm php8-dom php8-curl php8-json --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\ +RUN apk add php8 php8-fpm php8-dom php8-curl php8-json php8-apcu --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\ sed -i 's/^\s*listen = 127.0.0.1:9000/listen = \/run\/php8\/php-fpm8.sock/' ${WWW_CONFIG} &&\ sed -i 's/^\s*;\s*listen.owner = nobody/listen.owner = nginx/' ${WWW_CONFIG} &&\ sed -i 's/^\s*;\s*listen.group = nobody/listen.group = nginx/' ${WWW_CONFIG} &&\ From 11e11192a47b91e22e0f5726a060920b8ede03b5 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 18:57:23 +0100 Subject: [PATCH 6/9] Do not persist if apcu is not installed --- misc/cooldowns.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 70d0bbe9..1cc0fff5 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -1,10 +1,13 @@ Date: Fri, 25 Aug 2023 19:09:06 +0100 Subject: [PATCH 7/9] Fixed incorrect use of function_exists --- misc/cooldowns.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 1cc0fff5..0285f47e 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -1,12 +1,12 @@ Date: Fri, 25 Aug 2023 23:15:17 +0100 Subject: [PATCH 8/9] Fix missing quote in docker build script --- docker/attributes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/attributes.sh b/docker/attributes.sh index 900fd9d6..95a1c8ce 100755 --- a/docker/attributes.sh +++ b/docker/attributes.sh @@ -23,7 +23,7 @@ export CONFIG_GOOGLE_DOMAIN="${CONFIG_GOOGLE_DOMAIN:-"com"}" export CONFIG_GOOGLE_LANGUAGE_SITE="${CONFIG_GOOGLE_LANGUAGE_SITE:-"en"}" export CONFIG_GOOGLE_LANGUAGE_RESULTS="${CONFIG_GOOGLE_LANGUAGE_RESULTS:-"en"}" export CONFIG_GOOGLE_NUMBER_OF_RESULTS="${CONFIG_GOOGLE_NUMBER_OF_RESULTS:-"10"}" -export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true} +export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}" export CONFIG_INVIDIOUS_INSTANCE="${CONFIG_INVIDIOUS_INSTANCE:-"invidious.snopyta.org"}" export CONFIG_HIDDEN_SERVICE_SEARCH=${CONFIG_HIDDEN_SERVICE_SEARCH:-false} export CONFIG_DISABLE_BITTORRENT_SEARCH=${CONFIG_DISABLE_BITTORRENT_SEARCH:-false} From 98707900bfd6119a9a39183cda89de55939c44ec Mon Sep 17 00:00:00 2001 From: davidovski Date: Sat, 26 Aug 2023 00:41:25 +0100 Subject: [PATCH 9/9] Fix language config variable not working --- misc/search_engine.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/search_engine.php b/misc/search_engine.php index 75e87a5d..b76cdcb2 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -47,7 +47,7 @@ function load_opts() { $opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]); - $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? "")); + $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language)); $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; if (!$opts->instance_fallback) {