Browse Source

Mock HTTP requests in tests (#898)

Certain tests involving HTTP requests were sometimes hanging
indefinitely, so this uses Mockito for HTTP mocking. This seemingly
resolves the issue and makes these tests more reliable.

The existing can_fail_404_links test has been renamed to
can_fail_unresolved_links, to represent what actually occurs in the
test. The can_fail_404_links test now deals with a proper 404
response.

Just to be clear, the check_site test in the site component will
still create outgoing HTTP requests (due to the URLs used in the
test_site), so this commit only uses HTTP mocking where possible.
index-subcmd
Sam Ford Vincent Prouillet 4 years ago
parent
commit
11f7a6d114
5 changed files with 165 additions and 30 deletions
  1. +47
    -0
      Cargo.lock
  2. +3
    -0
      components/link_checker/Cargo.toml
  3. +79
    -19
      components/link_checker/src/lib.rs
  4. +3
    -0
      components/templates/Cargo.toml
  5. +33
    -11
      components/templates/src/global_fns/load_data.rs

+ 47
- 0
Cargo.lock View File

@@ -295,6 +295,15 @@ name = "arc-swap"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "assert-json-diff"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "atty"
version = "0.2.13"
@@ -471,6 +480,16 @@ name = "color_quant"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "colored"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "config"
version = "0.1.0"
@@ -678,6 +697,11 @@ name = "deunicode"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "difference"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "digest"
version = "0.8.1"
@@ -1295,6 +1319,7 @@ dependencies = [
"config 0.1.0",
"errors 0.1.0",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"mockito 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)",
"reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
]

@@ -1456,6 +1481,23 @@ dependencies = [
"ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "mockito"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"assert-json-diff 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"colored 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "native-tls"
version = "0.2.3"
@@ -2558,6 +2600,7 @@ dependencies = [
"imageproc 0.1.0",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"library 0.1.0",
"mockito 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pulldown-cmark 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3251,6 +3294,7 @@ dependencies = [
"checksum ammonia 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e266e1f4be5ffa05309f650e2586fe1d3ae6034eb24025a7ae1dfecc330823a"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum arc-swap 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff"
"checksum assert-json-diff 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9881d306dee755eccf052d652b774a6b2861e86b4772f555262130e58e4f81d2"
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
"checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
@@ -3273,6 +3317,7 @@ dependencies = [
"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
"checksum color_quant 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd"
"checksum colored 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "433e7ac7d511768127ed85b0c4947f47a254131e37864b2dc13f52aa32cd37e5"
"checksum const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b641a8c9867e341f3295564203b1c250eb8ce6cb6126e007941f78c4d2ed7fe"
"checksum const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c750ec12b83377637110d5a57f5ae08e895b06c4b16e2bdbf1a94ef717428c59"
"checksum cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
@@ -3294,6 +3339,7 @@ dependencies = [
"checksum deflate 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "707b6a7b384888a70c8d2e8650b3e60170dfc6a67bb4aa67b6dfca57af4bedb4"
"checksum derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a141330240c921ec6d074a3e188a7c7ef95668bb95e7d44fa0e5778ec2a7afe"
"checksum deunicode 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690"
"checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
"checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
"checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
@@ -3375,6 +3421,7 @@ dependencies = [
"checksum mio-extras 2.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19"
"checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
"checksum mockito 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4e524e85ea7c80559354217a6470c14abc2411802a9996aed1821559b9e28e3c"
"checksum native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e"
"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
"checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30"


+ 3
- 0
components/link_checker/Cargo.toml View File

@@ -10,3 +10,6 @@ lazy_static = "1"

config = { path = "../config" }
errors = { path = "../errors" }

[dev-dependencies]
mockito = "0.22"

+ 79
- 19
components/link_checker/src/lib.rs View File

@@ -140,29 +140,60 @@ fn check_page_for_anchor(url: &str, body: reqwest::Result<String>) -> Result<()>
#[cfg(test)]
mod tests {
use super::{check_page_for_anchor, check_url, has_anchor, LinkChecker, LINKS};
use mockito::mock;

#[test]
fn can_validate_ok_links() {
let url = "https://google.com";
let res = check_url(url, &LinkChecker::default());
assert!(res.is_valid());
assert!(LINKS.read().unwrap().get(url).is_some());
let res = check_url(url, &LinkChecker::default());
let url = format!("{}{}", mockito::server_url(), "/test");
let _m = mock("GET", "/test")
.with_header("content-type", "text/html")
.with_body(format!(
r#"<!DOCTYPE html>
<html>
<head>
<title>Test</title>
</head>
<body>
<a href="{}">Mock URL</a>
</body>
</html>
"#,
url
))
.create();

let res = check_url(&url, &LinkChecker::default());
assert!(res.is_valid());
assert!(LINKS.read().unwrap().get(&url).is_some());
}

#[test]
fn can_fail_404_links() {
fn can_fail_unresolved_links() {
let res = check_url("https://google.comys", &LinkChecker::default());
assert_eq!(res.is_valid(), false);
assert!(res.code.is_none());
assert!(res.error.is_some());
}

#[test]
fn can_fail_404_links() {
let _m = mock("GET", "/404")
.with_status(404)
.with_header("content-type", "text/plain")
.with_body("Not Found")
.create();

let url = format!("{}{}", mockito::server_url(), "/404");
let res = check_url(&url, &LinkChecker::default());
assert_eq!(res.is_valid(), false);
assert!(res.code.is_none());
assert!(res.error.is_some());
}

#[test]
fn can_validate_anchors() {
let url = "https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.collect";
let body = "<body><h3 id='method.collect'>collect</h3></body>".to_string();
let body = r#"<body><h3 id="method.collect">collect</h3></body>"#.to_string();
let res = check_page_for_anchor(url, Ok(body));
assert!(res.is_ok());
}
@@ -186,7 +217,7 @@ mod tests {
#[test]
fn can_fail_when_anchor_not_found() {
let url = "https://doc.rust-lang.org/std/iter/trait.Iterator.html#me";
let body = "<body><h3 id='method.collect'>collect</h3></body>".to_string();
let body = r#"<body><h3 id="method.collect">collect</h3></body>"#.to_string();
let res = check_page_for_anchor(url, Ok(body));
assert!(res.is_err());
}
@@ -221,21 +252,50 @@ mod tests {

#[test]
fn skip_anchor_prefixes() {
let config = LinkChecker {
skip_prefixes: vec![],
skip_anchor_prefixes: vec!["https://github.com/rust-lang/rust/blob/".to_owned()],
};
let ignore_url = format!("{}{}", mockito::server_url(), "/ignore/");
let config = LinkChecker { skip_prefixes: vec![], skip_anchor_prefixes: vec![ignore_url] };

let _m1 = mock("GET", "/ignore/test")
.with_header("content-type", "text/html")
.with_body(
r#"<!DOCTYPE html>
<html>
<head>
<title>Ignore</title>
</head>
<body>
<p id="existent"></p>
</body>
</html>
"#,
)
.create();

// anchor check is ignored because the url matches the prefix
let permalink = "https://github.com/rust-lang/rust/blob/c772948b687488a087356cb91432425662e034b9/src/librustc_back/target/mod.rs#L194-L214";
assert!(check_url(&permalink, &config).is_valid());
let ignore = format!("{}{}", mockito::server_url(), "/ignore/test#nonexistent");
assert!(check_url(&ignore, &config).is_valid());

let _m2 = mock("GET", "/test")
.with_header("content-type", "text/html")
.with_body(
r#"<!DOCTYPE html>
<html>
<head>
<title>Test</title>
</head>
<body>
<p id="existent"></p>
</body>
</html>
"#,
)
.create();

// other anchors are checked
let glossary = "https://help.github.com/en/articles/github-glossary#blame";
assert!(check_url(&glossary, &config).is_valid());
let existent = format!("{}{}", mockito::server_url(), "/test#existent");
assert!(check_url(&existent, &config).is_valid());

let glossary_invalid =
"https://help.github.com/en/articles/github-glossary#anchor-does-not-exist";
assert_eq!(check_url(&glossary_invalid, &config).is_valid(), false);
let nonexistent = format!("{}{}", mockito::server_url(), "/test#nonexistent");
assert_eq!(check_url(&nonexistent, &config).is_valid(), false);
}
}

+ 3
- 0
components/templates/Cargo.toml View File

@@ -21,3 +21,6 @@ utils = { path = "../utils" }
library = { path = "../library" }
config = { path = "../config" }
imageproc = { path = "../imageproc" }

[dev-dependencies]
mockito = "0.22"

+ 33
- 11
components/templates/src/global_fns/load_data.rs View File

@@ -321,6 +321,7 @@ mod tests {
use std::collections::HashMap;
use std::path::PathBuf;

use mockito::mock;
use serde_json::json;
use tera::{to_value, Function};

@@ -365,10 +366,14 @@ mod tests {

#[test]
fn calculates_cache_key_for_url() {
let cache_key =
DataSource::Url("https://api.github.com/repos/getzola/zola".parse().unwrap())
.get_cache_key(&OutputFormat::Plain);
assert_eq!(cache_key, 8916756616423791754);
let _m = mock("GET", "/test")
.with_header("content-type", "text/plain")
.with_body("Test")
.create();

let url = format!("{}{}", mockito::server_url(), "/test");
let cache_key = DataSource::Url(url.parse().unwrap()).get_cache_key(&OutputFormat::Plain);
assert_eq!(cache_key, 12502656262443320092);
}

#[test]
@@ -391,28 +396,45 @@ mod tests {

#[test]
fn can_load_remote_data() {
let _m = mock("GET", "/json")
.with_header("content-type", "application/json")
.with_body(
r#"{
"test": {
"foo": "bar"
}
}
"#,
)
.create();

let url = format!("{}{}", mockito::server_url(), "/json");
let static_fn = LoadData::new(PathBuf::new());
let mut args = HashMap::new();
args.insert("url".to_string(), to_value("https://httpbin.org/json").unwrap());
args.insert("url".to_string(), to_value(&url).unwrap());
args.insert("format".to_string(), to_value("json").unwrap());
let result = static_fn.call(&args).unwrap();
assert_eq!(
result.get("slideshow").unwrap().get("title").unwrap(),
&to_value("Sample Slide Show").unwrap()
);
assert_eq!(result.get("test").unwrap().get("foo").unwrap(), &to_value("bar").unwrap());
}

#[test]
fn fails_when_request_404s() {
let _m = mock("GET", "/404")
.with_status(404)
.with_header("content-type", "text/plain")
.with_body("Not Found")
.create();

let url = format!("{}{}", mockito::server_url(), "/404");
let static_fn = LoadData::new(PathBuf::new());
let mut args = HashMap::new();
args.insert("url".to_string(), to_value("https://httpbin.org/status/404/").unwrap());
args.insert("url".to_string(), to_value(&url).unwrap());
args.insert("format".to_string(), to_value("json").unwrap());
let result = static_fn.call(&args);
assert!(result.is_err());
assert_eq!(
result.unwrap_err().to_string(),
"Failed to request https://httpbin.org/status/404/: 404 Not Found"
format!("Failed to request {}: 404 Not Found", url)
);
}



Loading…
Cancel
Save