|
|
|
@ -35,114 +35,6 @@
|
|
|
|
|
: $str;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Check whether URL is reachable, downgrading to HTTP if needed.
|
|
|
|
|
* @param string $url URL to check.
|
|
|
|
|
* @param int $retries [optional] Number of connection attempts.
|
|
|
|
|
* @return bool Whether or not the server responded with a non-5XX HTTP code.
|
|
|
|
|
*/
|
|
|
|
|
function url_is_reachable(string $url, int $retries = 4): bool {
|
|
|
|
|
$retcode = curl_get_response_downgrade(
|
|
|
|
|
$url, retries: $retries,
|
|
|
|
|
curlopts: [CURLOPT_NOBODY => true], stop_on_codes: [404]
|
|
|
|
|
)[0];
|
|
|
|
|
return $retcode != 0 && floor($retcode / 100) != 5 ;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Fetch URL repeatedly to obtain contents, downgrading to HTTP if needed.
|
|
|
|
|
* @param string $url URL to fetch.
|
|
|
|
|
* @param int $retries [optional] Number of connection attempts.
|
|
|
|
|
* @param int[] $stop_on_codes [optional]
|
|
|
|
|
* If one of these HTTP codes is encountered, fetching stops early.
|
|
|
|
|
* @return array Fetched contents (if applicable),
|
|
|
|
|
* and whether a downgrade to HTTP took place.
|
|
|
|
|
* A code of 0 corresponds to an unreachable host.
|
|
|
|
|
*/
|
|
|
|
|
function curl_get_contents_downgrade(string $url, int $retries = 4, $stop_on_codes = [404]) {
|
|
|
|
|
list($retcode, $content, $downgrade) = curl_get_response_downgrade($url, $retries, $stop_on_codes);
|
|
|
|
|
return [$retcode == 200 ? $content : null, $downgrade];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Fetch URL repeatedly to obtain URL contents.
|
|
|
|
|
* @param string $url URL to fetch.
|
|
|
|
|
* @param int $retries [optional] Number of connection attempts.
|
|
|
|
|
* @param int[] $stop_on_codes [optional]
|
|
|
|
|
* If one of these HTTP codes is encountered, fetching stops early.
|
|
|
|
|
* @return ?string Fetched contents, if applicable.
|
|
|
|
|
*/
|
|
|
|
|
function curl_get_contents(string $url, int $retries = 4, $stop_on_codes = [404]): ?string {
|
|
|
|
|
return curl_get_response($url, retries: $retries, stop_on_codes: $stop_on_codes)[1];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Fetch URL repeatedly, downgrading to HTTP if needed.
|
|
|
|
|
* @param string $url URL to fetch.
|
|
|
|
|
* @param int $retries [optional] Number of connection attempts.
|
|
|
|
|
* @param int[] $stop_on_codes [optional]
|
|
|
|
|
* If one of these HTTP codes is encountered, fetching stops early.
|
|
|
|
|
* @param int[] $curlopts Associative array of options for `curl_setopt`.
|
|
|
|
|
* @return array Return code, fetched contents (if applicable),
|
|
|
|
|
* and whether a downgrade to HTTP took place.
|
|
|
|
|
* A code of 0 corresponds to an unreachable host.
|
|
|
|
|
*/
|
|
|
|
|
function curl_get_response_downgrade(
|
|
|
|
|
string $url, $retries = 4, $stop_on_codes = [404], $curlopts = []
|
|
|
|
|
) {
|
|
|
|
|
$scheme = parse_url($url, PHP_URL_SCHEME);
|
|
|
|
|
if ($scheme == "https") {
|
|
|
|
|
list($retcode, $content) = curl_get_response($url, floor($retries / 2), $stop_on_codes, $curlopts);
|
|
|
|
|
if ($retcode == 200) return [$retcode, $content, false];
|
|
|
|
|
log_debug("Downgrading to HTTP");
|
|
|
|
|
list($retcode, $content) = curl_get_response(
|
|
|
|
|
substr_replace($url, "http:", 0, strlen("https:")),
|
|
|
|
|
ceil($retries / 2),
|
|
|
|
|
$stop_on_codes, $curlopts
|
|
|
|
|
);
|
|
|
|
|
return [$retcode, $content, true];
|
|
|
|
|
} else {
|
|
|
|
|
list($retcode, $content) = curl_get_response($url, $retries, $stop_on_codes, $curlopts);
|
|
|
|
|
return [$retcode, $content, false];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Low default retries value so this doesn't run for 30 minutes
|
|
|
|
|
// FIXME: Does not seem to handle 308's, behaviour not transparent.
|
|
|
|
|
// TODO: Parallelize & use in CommunityServer::poll_reachable()
|
|
|
|
|
/**
|
|
|
|
|
* Fetch URL repeatedly to obtain a valid response.
|
|
|
|
|
* @param string $url URL to fetch.
|
|
|
|
|
* @param int $retries Number of connection attempts.
|
|
|
|
|
* @param string[] $stop_on_codes [optional]
|
|
|
|
|
* If one of these HTTP codes is encountered, fetching stops early.
|
|
|
|
|
* @param int[] $curlopts Associative array of options for `curl_setopt`.
|
|
|
|
|
* @return array Return code and fetched contents, if applicable. A code of 0 corresponds
|
|
|
|
|
* to an unreachable host.
|
|
|
|
|
*/
|
|
|
|
|
function curl_get_response(string $url, int $retries, $stop_on_codes = [404], $curlopts = []) {
|
|
|
|
|
global $CURL_RETRY_SLEEP;
|
|
|
|
|
// use separate timeouts to reliably get data from Chinese server with repeated tries
|
|
|
|
|
|
|
|
|
|
$contents = false;
|
|
|
|
|
$retcode = -1;
|
|
|
|
|
|
|
|
|
|
for ($counter = 1; $counter <= $retries; $counter++) {
|
|
|
|
|
$curl = make_curl_handle($url, $curlopts);
|
|
|
|
|
$contents = curl_exec($curl);
|
|
|
|
|
$retcode = curl_getinfo($curl, CURLINFO_HTTP_CODE);
|
|
|
|
|
|
|
|
|
|
curl_close($curl);
|
|
|
|
|
|
|
|
|
|
log_debug("Attempt #" . $counter . " for " . $url . " returned code " . $retcode . ".");
|
|
|
|
|
if ($contents != null || $retcode == 200 || in_array($retcode, $stop_on_codes)) break;
|
|
|
|
|
sleep($CURL_RETRY_SLEEP / 1E3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return [$retcode, $retcode == 200 ? $contents : false];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function make_curl_handle(string $url, $curlopts = []) {
|
|
|
|
|
global $CURL_CONNECT_TIMEOUT_MS, $CURL_TIMEOUT_MS;
|
|
|
|
|
|
|
|
|
|