From 6a5ace62fcfce45a8e7ea3b2006577e24d010371 Mon Sep 17 00:00:00 2001 From: Thomas Hurst Date: Sat, 25 Aug 2018 16:50:01 +0100 Subject: [PATCH] link_checker: Set Accept header As mentioned in #381, crates.io 404's any request without an Accept: text/html header. It 200's any request with one, but at least false-successes don't prevent checking any other links. This also makes it easier to add a custom User-Agent if desired. rustfmt and fix a clippy nit (unnecessary return) while I'm here. --- components/link_checker/src/lib.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/components/link_checker/src/lib.rs b/components/link_checker/src/lib.rs index 4aa2d81..1aac769 100644 --- a/components/link_checker/src/lib.rs +++ b/components/link_checker/src/lib.rs @@ -2,11 +2,11 @@ extern crate reqwest; #[macro_use] extern crate lazy_static; +use reqwest::header::{qitem, Accept, Headers}; +use reqwest::{mime, StatusCode}; use std::collections::HashMap; use std::error::Error; use std::sync::{Arc, RwLock}; -use reqwest::StatusCode; - #[derive(Clone, Debug, PartialEq)] pub struct LinkResult { @@ -54,19 +54,30 @@ pub fn check_url(url: &str) -> LinkResult { } } + let mut headers = Headers::new(); + headers.set(Accept(vec![qitem(mime::TEXT_HTML), qitem(mime::STAR_STAR)])); + + let client = reqwest::Client::new(); + // Need to actually do the link checking - let res = match reqwest::get(url) { - Ok(response) => LinkResult { code: Some(response.status()), error: None }, - Err(e) => LinkResult { code: None, error: Some(e.description().to_string()) }, + let res = match client.get(url).headers(headers).send() { + Ok(response) => LinkResult { + code: Some(response.status()), + error: None, + }, + Err(e) => LinkResult { + code: None, + error: Some(e.description().to_string()), + }, }; LINKS.write().unwrap().insert(url.to_string(), res.clone()); - return res; + res } #[cfg(test)] mod tests { - use super::{LINKS, check_url}; + use super::{check_url, LINKS}; #[test] fn can_validate_ok_links() {