diff --git a/README.md b/README.md index f56505af..44e781ad 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ You can access the full list of LibreX and LibreY instances on one of the follow ### About LibreY LibreY gives you text results from DuckDuckGo or Google, images from Qwant, and torrents from i.e. Ahmia and popular torrent sites without spying on you. -
LibreY doesn't save any type of data about the user, there are no logs (except NGINX logs if the host sets them), no caches. +
LibreY doesn't save **any** type of data about the user, there are no logs (except NGINX logs if the host sets them). ### LibreY compared to other metasearch engines diff --git a/config.php.example b/config.php.example index f906c724..567e839e 100644 --- a/config.php.example +++ b/config.php.example @@ -17,8 +17,11 @@ "disable_hidden_service_search" => false, // Fallback to another librex instance if google search fails - // This may greatly increase the time it takes to get a result and in some cases results in 504 errors - "instance_fallback" => false, + // This may greatly increase the time it takes to get a result, if a direct search is not possible + "instance_fallback" => true, + + // how long in minutes to put google/other instances on cooldown if they aren't responding + "request_cooldown" => 25, /* Preset privacy friendly frontends for users, these can be overwritten by users in the settings @@ -148,7 +151,8 @@ CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, CURLOPT_MAXREDIRS => 5, CURLOPT_TIMEOUT => 3, - CURLOPT_VERBOSE => false + CURLOPT_VERBOSE => false, + CURLOPT_FOLLOWLOCATION => true ) ); ?> diff --git a/docker/attributes.sh b/docker/attributes.sh index 900fd9d6..95a1c8ce 100755 --- a/docker/attributes.sh +++ b/docker/attributes.sh @@ -23,7 +23,7 @@ export CONFIG_GOOGLE_DOMAIN="${CONFIG_GOOGLE_DOMAIN:-"com"}" export CONFIG_GOOGLE_LANGUAGE_SITE="${CONFIG_GOOGLE_LANGUAGE_SITE:-"en"}" export CONFIG_GOOGLE_LANGUAGE_RESULTS="${CONFIG_GOOGLE_LANGUAGE_RESULTS:-"en"}" export CONFIG_GOOGLE_NUMBER_OF_RESULTS="${CONFIG_GOOGLE_NUMBER_OF_RESULTS:-"10"}" -export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true} +export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}" export CONFIG_INVIDIOUS_INSTANCE="${CONFIG_INVIDIOUS_INSTANCE:-"invidious.snopyta.org"}" export CONFIG_HIDDEN_SERVICE_SEARCH=${CONFIG_HIDDEN_SERVICE_SEARCH:-false} export CONFIG_DISABLE_BITTORRENT_SEARCH=${CONFIG_DISABLE_BITTORRENT_SEARCH:-false} diff --git a/docker/php/php.dockerfile b/docker/php/php.dockerfile index 5a522658..87351377 100644 --- a/docker/php/php.dockerfile +++ b/docker/php/php.dockerfile @@ -56,7 +56,7 @@ ENV CURLOPT_VERBOSE=true # Install PHP-FPM using Alpine's package manager, apk # Configure PHP-FPM to listen on a Unix socket instead of a TCP port, which is more secure and efficient -RUN apk add php8 php8-fpm php8-dom php8-curl php8-json --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\ +RUN apk add php8 php8-fpm php8-dom php8-curl php8-json php8-apcu --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\ sed -i 's/^\s*listen = 127.0.0.1:9000/listen = \/run\/php8\/php-fpm8.sock/' ${WWW_CONFIG} &&\ sed -i 's/^\s*;\s*listen.owner = nobody/listen.owner = nginx/' ${WWW_CONFIG} &&\ sed -i 's/^\s*;\s*listen.group = nobody/listen.group = nginx/' ${WWW_CONFIG} &&\ diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 7d5e6a10..4ff13b00 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -15,25 +15,29 @@ public function get_results() { $response = json_decode($response, true); if (!$response) return array(); - + return array_values($response); } } - - function get_librex_results($opts) { - if (!$opts->do_fallback) - return array(); - + function load_instances($cooldowns) { $instances_json = json_decode(file_get_contents("instances.json"), true); if (empty($instances_json["instances"])) return array(); - // TODO pick instances which aren't on cooldown - $instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet']))); + $instances = array_filter($instances, fn($n) => !has_cooldown($n, $cooldowns)); shuffle($instances); + return $instances; + } + + function get_librex_results($opts) { + if (!$opts->do_fallback) + return array(); + + $cooldowns = $opts->cooldowns; + $instances = load_instances($cooldowns); $results = array(); $tries = 0; @@ -52,12 +56,13 @@ function get_librex_results($opts) { if (count($results) > 1) return $results; - } while ( !empty($instances)); + // on fail then do this + $timeout = ($opts->request_cooldown ?? "1") * 60; + $cooldowns = set_cooldown($instance, $timeout, $cooldowns); - if (empty($instances)) - return array(); + } while (!empty($instances)); - return array_values($results); + return array(); } ?> diff --git a/engines/text/google.php b/engines/text/google.php index d43ff1d2..45c871d8 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -1,7 +1,6 @@ query)); $results = array(); diff --git a/engines/text/text.php b/engines/text/text.php index ea983beb..26af3ae4 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -5,19 +5,23 @@ public function __construct($opts, $mh) { $this->page = $opts->page; $this->opts = $opts; - $engine = $opts->preferred_engines["text"] ?? "google"; + $this->engine = $opts->preferred_engines["text"] ?? "google"; $query_parts = explode(" ", $this->query); $last_word_query = end($query_parts); if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") check_ddg_bang($this->query, $opts); - if ($engine == "google") { + if (has_cooldown($this->engine, $this->opts->cooldowns)) + return; + + if ($this->engine == "google") { + require "engines/text/google.php"; $this->engine_request = new GoogleRequest($opts, $mh); } - if ($engine == "duckduckgo") { + if ($this->engine == "duckduckgo") { require "engines/text/duckduckgo.php"; $this->engine_request = new DuckDuckGoRequest($opts, $mh); } @@ -27,6 +31,9 @@ public function __construct($opts, $mh) { } public function get_results() { + if (!$this->engine_request) + return array(); + $results = $this->engine_request->get_results(); if ($this->special_request) { @@ -36,6 +43,9 @@ public function get_results() { $results = array_merge(array($special_result), $results); } + if (count($results) <= 1) + set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns); + return $results; } diff --git a/misc/cooldowns.php b/misc/cooldowns.php new file mode 100644 index 00000000..0285f47e --- /dev/null +++ b/misc/cooldowns.php @@ -0,0 +1,22 @@ + time(); + } +?> diff --git a/misc/search_engine.php b/misc/search_engine.php index 1c1c1002..b76cdcb2 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -33,6 +33,8 @@ static public function print_results($results){} function load_opts() { $opts = require "config.php"; + $opts->request_cooldown ??= 25; + $opts->query = trim($_REQUEST["q"] ?? ""); $opts->type = (int) ($_REQUEST["t"] ?? 0); $opts->page = (int) ($_REQUEST["p"] ?? 0); @@ -45,7 +47,7 @@ function load_opts() { $opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]); - $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? "")); + $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language)); $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; if (!$opts->instance_fallback) { @@ -57,6 +59,7 @@ function load_opts() { foreach (array_keys($opts->frontends ?? array()) as $frontend) { $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"]; } + return $opts; } @@ -110,6 +113,9 @@ function init_search($opts, $mh) { } function fetch_search_results($opts, $do_print) { + require "misc/cooldowns.php"; + $opts->cooldowns = load_cooldowns(); + $start_time = microtime(true); $mh = curl_multi_init(); $search_category = init_search($opts, $mh);