Perform validation of redirect URLs during the redirect process.

Previously, validation was only done after all redirects and the final request had completed.  This approach ensures all redirects are to URLs that pass extended validation.
This commit is contained in:
wn_ 2023-12-29 00:31:03 +00:00
parent 51cd02fc3e
commit 91a91dac15
2 changed files with 31 additions and 7 deletions

View File

@ -1,4 +1,9 @@
<?php <?php
use Psr\Http\Message\RequestInterface;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\UriInterface;
class UrlHelper { class UrlHelper {
const EXTRA_HREF_SCHEMES = [ const EXTRA_HREF_SCHEMES = [
"magnet", "magnet",
@ -312,13 +317,28 @@ class UrlHelper {
$req_options = [ $req_options = [
GuzzleHttp\RequestOptions::CONNECT_TIMEOUT => $timeout ?: Config::get(Config::FILE_FETCH_CONNECT_TIMEOUT), GuzzleHttp\RequestOptions::CONNECT_TIMEOUT => $timeout ?: Config::get(Config::FILE_FETCH_CONNECT_TIMEOUT),
GuzzleHttp\RequestOptions::TIMEOUT => $timeout ?: Config::get(Config::FILE_FETCH_TIMEOUT), GuzzleHttp\RequestOptions::TIMEOUT => $timeout ?: Config::get(Config::FILE_FETCH_TIMEOUT),
GuzzleHttp\RequestOptions::ALLOW_REDIRECTS => $followlocation ? ['max' => 20, 'track_redirects' => true] : false,
GuzzleHttp\RequestOptions::HEADERS => [ GuzzleHttp\RequestOptions::HEADERS => [
'User-Agent' => $useragent ?: Config::get_user_agent(), 'User-Agent' => $useragent ?: Config::get_user_agent(),
], ],
'curl' => [], 'curl' => [],
]; ];
if ($followlocation) {
$req_options[GuzzleHttp\RequestOptions::ALLOW_REDIRECTS] = [
'max' => 20,
'track_redirects' => true,
'on_redirect' => function(RequestInterface $request, ResponseInterface $response, UriInterface $uri) {
if (!self::validate($uri, true)) {
self::$fetch_effective_url = (string) $uri;
throw new GuzzleHttp\Exception\RequestException('URL received during redirection failed extended validation.',
$request, $response);
}
},
];
} else {
$req_options[GuzzleHttp\RequestOptions::ALLOW_REDIRECTS] = false;
}
if ($last_modified && !$post_query) if ($last_modified && !$post_query)
$req_options[GuzzleHttp\RequestOptions::HEADERS]['If-Modified-Since'] = $last_modified; $req_options[GuzzleHttp\RequestOptions::HEADERS]['If-Modified-Since'] = $last_modified;
@ -355,7 +375,7 @@ class UrlHelper {
}; };
# Alternative/supplement to `progress` checking # Alternative/supplement to `progress` checking
$req_options[GuzzleHttp\RequestOptions::ON_HEADERS] = function(Psr\Http\Message\ResponseInterface $response) use(&$max_size, $url) { $req_options[GuzzleHttp\RequestOptions::ON_HEADERS] = function(ResponseInterface $response) use(&$max_size, $url) {
$content_length = $response->getHeaderLine('Content-Length'); $content_length = $response->getHeaderLine('Content-Length');
if ($content_length > $max_size) { if ($content_length > $max_size) {
Debug::log("[UrlHelper] fetch error: server indicated (via 'Content-Length: {$content_length}') max size of $max_size bytes " . Debug::log("[UrlHelper] fetch error: server indicated (via 'Content-Length: {$content_length}') max size of $max_size bytes " .
@ -426,6 +446,7 @@ class UrlHelper {
$history_header = $response->getHeader(GuzzleHttp\RedirectMiddleware::HISTORY_HEADER); $history_header = $response->getHeader(GuzzleHttp\RedirectMiddleware::HISTORY_HEADER);
self::$fetch_effective_url = $history_header ? end($history_header) : $url; self::$fetch_effective_url = $history_header ? end($history_header) : $url;
// This shouldn't be necessary given the checks that occur during potential redirects, but we'll do it anyway.
if (!self::validate(self::$fetch_effective_url, true)) { if (!self::validate(self::$fetch_effective_url, true)) {
self::$fetch_last_error = "URL received after redirection failed extended validation."; self::$fetch_last_error = "URL received after redirection failed extended validation.";
$span->setAttribute('error', self::$fetch_last_error); $span->setAttribute('error', self::$fetch_last_error);

View File

@ -74,6 +74,7 @@ final class UrlHelperTest extends TestCase {
$result = UrlHelper::fetch('https://www.example.com'); $result = UrlHelper::fetch('https://www.example.com');
$this->assertEquals(200, UrlHelper::$fetch_last_error_code); $this->assertEquals(200, UrlHelper::$fetch_last_error_code);
$this->assertEquals('Hello, World', $result); $this->assertEquals('Hello, World', $result);
$mock->reset();
foreach (['ftp://ftp.example.com', 'http://127.0.0.1', 'blah', '', 42, null] as $url) { foreach (['ftp://ftp.example.com', 'http://127.0.0.1', 'blah', '', 42, null] as $url) {
$result = UrlHelper::fetch($url); $result = UrlHelper::fetch($url);
@ -83,24 +84,25 @@ final class UrlHelperTest extends TestCase {
$mock->append(new Response(200, ['Content-Length' => (string) PHP_INT_MAX])); $mock->append(new Response(200, ['Content-Length' => (string) PHP_INT_MAX]));
$result = UrlHelper::fetch('https://www.example.com/very-large-content-length'); $result = UrlHelper::fetch('https://www.example.com/very-large-content-length');
$this->assertFalse($result); $this->assertFalse($result);
$mock->reset();
$mock->append(new Response(301, ['Location' => 'https://www.example.com'])); $mock->append(new Response(301, ['Location' => 'https://www.example.com']));
$result = UrlHelper::fetch(['url' => 'https://example.com', 'followlocation' => false]); $result = UrlHelper::fetch(['url' => 'https://example.com', 'followlocation' => false]);
$this->assertFalse($result); $this->assertFalse($result);
$mock->reset();
$mock->append( $mock->append(new Response(301, ['Location' => 'http://127.0.0.1']));
new Response(301, ['Location' => 'http://127.0.0.1']),
new Response(200, [], 'Hello, World'),
);
$result = UrlHelper::fetch(['url' => 'https://example.com', 'followlocation' => true]); $result = UrlHelper::fetch(['url' => 'https://example.com', 'followlocation' => true]);
$this->assertFalse($result); $this->assertFalse($result);
$this->assertEquals('URL received after redirection failed extended validation.', UrlHelper::$fetch_last_error); $this->assertMatchesRegularExpression('%failed extended validation%', UrlHelper::$fetch_last_error);
$this->assertEquals('http://127.0.0.1', UrlHelper::$fetch_effective_url); $this->assertEquals('http://127.0.0.1', UrlHelper::$fetch_effective_url);
$mock->reset();
$mock->append(new Response(200, [], '')); $mock->append(new Response(200, [], ''));
$result = UrlHelper::fetch('https://www.example.com'); $result = UrlHelper::fetch('https://www.example.com');
$this->assertFalse($result); $this->assertFalse($result);
$this->assertEquals('Successful response, but no content was received.', UrlHelper::$fetch_last_error); $this->assertEquals('Successful response, but no content was received.', UrlHelper::$fetch_last_error);
$mock->reset();
// Fake a 403 for basic auth and success with `CURLAUTH_ANY` in the retry attempt // Fake a 403 for basic auth and success with `CURLAUTH_ANY` in the retry attempt
$mock->append( $mock->append(
@ -116,5 +118,6 @@ final class UrlHelperTest extends TestCase {
$this->assertEquals(200, UrlHelper::$fetch_last_error_code); $this->assertEquals(200, UrlHelper::$fetch_last_error_code);
$this->assertEquals('Hello, World', $result); $this->assertEquals('Hello, World', $result);
$this->assertEquals($mock->getLastOptions()['curl'][\CURLOPT_HTTPAUTH], \CURLAUTH_ANY); $this->assertEquals($mock->getLastOptions()['curl'][\CURLOPT_HTTPAUTH], \CURLAUTH_ANY);
$mock->reset();
} }
} }