Redesign routing benchmarks.

The new benchmarks use routes from real-world project. This is much more
realistic than the previous benchmarks.

The new benchmarks use `criterion` and exist in their own Cargo project.
This commit is contained in:
Sergio Benitez 2021-03-26 16:39:24 -07:00
parent 20605dac14
commit 3119e6f453
10 changed files with 330 additions and 262 deletions

12
benchmarks/Cargo.toml Normal file
View File

@ -0,0 +1,12 @@
[package]
name = "rocket-benchmarks"
version = "0.0.0"
edition = "2018"
publish = false
[workspace]
[dependencies]
rocket = { path = "../core/lib/" }
criterion = "0.3"
criterion-macro = "0.3"

11
benchmarks/src/main.rs Normal file
View File

@ -0,0 +1,11 @@
#![feature(custom_test_frameworks)]
#![test_runner(criterion::runner)]
#[cfg_attr(test, macro_use)]
extern crate criterion_macro;
#[cfg(test)] mod routing;
pub fn main() {
eprintln!("help: cargo bench");
}

119
benchmarks/src/routing.rs Normal file
View File

@ -0,0 +1,119 @@
use criterion::Criterion;
use rocket::{handler, Request, Data, Route, Config};
use rocket::http::{Method, RawStr, ContentType, Accept, Status};
use rocket::local::blocking::{Client, LocalRequest};
fn dummy_handler<'r>(req: &'r Request, _: Data) -> handler::HandlerFuture<'r> {
handler::Outcome::from(req, ()).pin()
}
fn parse_routes_table(table: &str) -> Vec<Route> {
let mut routes = vec![];
for line in table.split("\n").filter(|s| !s.is_empty()) {
let mut components = line.split(" ");
let method: Method = components.next().expect("c").parse().expect("method");
let uri: &str = components.next().unwrap();
let (mut rank, mut name, mut format) = (None, None, None);
for component in components {
match component {
c if c.starts_with('[') => rank = c.trim_matches(&['[', ']'][..]).parse().ok(),
c if c.starts_with('(') => name = Some(c.trim_matches(&['(', ')'][..])),
c => format = c.parse().ok(),
}
}
let mut route = Route::new(method, uri, dummy_handler);
if let Some(rank) = rank {
route.rank = rank;
}
route.format = format;
route.name = name.map(|s| s.to_string().into());
routes.push(route);
}
routes
}
fn generate_matching_requests<'c>(client: &'c Client, routes: &[Route]) -> Vec<LocalRequest<'c>> {
fn staticify_segment(segment: &RawStr) -> &str {
segment.as_str().trim_matches(&['<', '>', '.', '_'][..])
}
fn request_for_route<'c>(client: &'c Client, route: &Route) -> LocalRequest<'c> {
let path = route.uri.raw_path_segments()
.map(staticify_segment)
.collect::<Vec<_>>()
.join("/");
let query = route.uri.raw_query_segments()
.map(staticify_segment)
.collect::<Vec<_>>()
.join("&");
let uri = format!("/{}?{}", path, query);
let mut req = client.req(route.method, uri);
if let Some(ref format) = route.format {
if route.method.supports_payload() {
req.add_header(ContentType::from(format.clone()));
} else {
req.add_header(Accept::from(format.clone()));
}
}
req
}
routes.iter()
.map(|route| request_for_route(client, route))
.collect()
}
fn client(routes: Vec<Route>) -> Client {
let config = Config {
profile: Config::RELEASE_PROFILE,
log_level: rocket::config::LogLevel::Off,
cli_colors: false,
ctrlc: false,
..Default::default()
};
match Client::untracked(rocket::custom(config).mount("/", routes)) {
Ok(client) => client,
Err(e) => {
drop(e);
panic!("bad launch")
}
}
}
#[criterion]
pub fn bench_rust_lang_routes(c: &mut Criterion) {
let table = include_str!("../static/rust-lang.routes");
let routes = parse_routes_table(table);
let client = client(routes.clone());
let requests = generate_matching_requests(&client, &routes);
c.bench_function("rust-lang.routes", |b| b.iter(|| {
for request in requests.clone() {
let response = request.dispatch();
assert_eq!(response.status(), Status::Ok);
}
}));
}
#[criterion]
pub fn bench_bitwarden_routes(c: &mut Criterion) {
let table = include_str!("../static/bitwarden_rs.routes");
let routes = parse_routes_table(table);
let client = client(routes.clone());
let requests = generate_matching_requests(&client, &routes);
c.bench_function("bitwarden_rs.routes", |b| b.iter(|| {
for request in requests.clone() {
let response = request.dispatch();
assert_eq!(response.status(), Status::Ok);
}
}));
}

View File

@ -0,0 +1,157 @@
GET /attachments/<uuid>/<file..> (attachments)
GET /alive (alive)
GET /bwrs_static/<filename> (static_files)
POST /api/accounts/register (register)
GET /api/accounts/profile (profile)
PUT /api/accounts/profile (put_profile)
POST /api/accounts/profile (post_profile)
GET /api/users/<uuid>/public-key (get_public_keys)
POST /api/accounts/keys (post_keys)
POST /api/accounts/password (post_password)
POST /api/accounts/kdf (post_kdf)
POST /api/accounts/key (post_rotatekey)
POST /api/accounts/security-stamp (post_sstamp)
POST /api/accounts/email-token (post_email_token)
POST /api/accounts/email (post_email)
POST /api/accounts/verify-email (post_verify_email)
POST /api/accounts/verify-email-token (post_verify_email_token)
POST /api/accounts/delete-recover (post_delete_recover)
POST /api/accounts/delete-recover-token (post_delete_recover_token)
DELETE /api/accounts (delete_account)
POST /api/accounts/delete (post_delete_account)
GET /api/accounts/revision-date (revision_date)
POST /api/accounts/password-hint (password_hint)
POST /api/accounts/prelogin (prelogin)
POST /api/accounts/verify-password (verify_password)
GET /api/sync?<data..> (sync)
GET /api/ciphers (get_ciphers)
GET /api/ciphers/<uuid> (get_cipher)
GET /api/ciphers/<uuid>/admin (get_cipher_admin)
GET /api/ciphers/<uuid>/details (get_cipher_details)
POST /api/ciphers (post_ciphers)
PUT /api/ciphers/<uuid>/admin (put_cipher_admin)
POST /api/ciphers/admin (post_ciphers_admin)
POST /api/ciphers/create (post_ciphers_create)
POST /api/ciphers/import (post_ciphers_import)
POST /api/ciphers/<uuid>/attachment multipart/form-data (post_attachment)
POST /api/ciphers/<uuid>/attachment-admin multipart/form-data (post_attachment_admin)
POST /api/ciphers/<uuid>/attachment/<attachment_id>/share multipart/form-data (post_attachment_share)
POST /api/ciphers/<uuid>/attachment/<attachment_id>/delete (delete_attachment_post)
POST /api/ciphers/<uuid>/attachment/<attachment_id>/delete-admin (delete_attachment_post_admin)
DELETE /api/ciphers/<uuid>/attachment/<attachment_id> (delete_attachment)
DELETE /api/ciphers/<uuid>/attachment/<attachment_id>/admin (delete_attachment_admin)
POST /api/ciphers/<uuid>/admin (post_cipher_admin)
POST /api/ciphers/<uuid>/share (post_cipher_share)
PUT /api/ciphers/<uuid>/share (put_cipher_share)
PUT /api/ciphers/share (put_cipher_share_selected)
POST /api/ciphers/<uuid> (post_cipher)
PUT /api/ciphers/<uuid> (put_cipher)
POST /api/ciphers/<uuid>/delete (delete_cipher_post)
POST /api/ciphers/<uuid>/delete-admin (delete_cipher_post_admin)
PUT /api/ciphers/<uuid>/delete (delete_cipher_put)
PUT /api/ciphers/<uuid>/delete-admin (delete_cipher_put_admin)
DELETE /api/ciphers/<uuid> (delete_cipher)
DELETE /api/ciphers/<uuid>/admin (delete_cipher_admin)
DELETE /api/ciphers (delete_cipher_selected)
POST /api/ciphers/delete (delete_cipher_selected_post)
PUT /api/ciphers/delete (delete_cipher_selected_put)
DELETE /api/ciphers/admin (delete_cipher_selected_admin)
POST /api/ciphers/delete-admin (delete_cipher_selected_post_admin)
PUT /api/ciphers/delete-admin (delete_cipher_selected_put_admin)
PUT /api/ciphers/<uuid>/restore (restore_cipher_put)
PUT /api/ciphers/<uuid>/restore-admin (restore_cipher_put_admin)
PUT /api/ciphers/restore (restore_cipher_selected)
POST /api/ciphers/purge?<organization..> (delete_all)
POST /api/ciphers/move (move_cipher_selected)
PUT /api/ciphers/move (move_cipher_selected_put)
PUT /api/ciphers/<uuid>/collections (put_collections_update)
POST /api/ciphers/<uuid>/collections (post_collections_update)
POST /api/ciphers/<uuid>/collections-admin (post_collections_admin)
PUT /api/ciphers/<uuid>/collections-admin (put_collections_admin)
GET /api/folders (get_folders)
GET /api/folders/<uuid> (get_folder)
POST /api/folders (post_folders)
POST /api/folders/<uuid> (post_folder)
PUT /api/folders/<uuid> (put_folder)
POST /api/folders/<uuid>/delete (delete_folder_post)
DELETE /api/folders/<uuid> (delete_folder)
GET /api/organizations/<org_id> (get_organization)
POST /api/organizations (create_organization)
DELETE /api/organizations/<org_id> (delete_organization)
POST /api/organizations/<org_id>/delete (post_delete_organization)
POST /api/organizations/<org_id>/leave (leave_organization)
GET /api/collections (get_user_collections)
GET /api/organizations/<org_id>/collections (get_org_collections)
GET /api/organizations/<org_id>/collections/<coll_id>/details (get_org_collection_detail)
GET /api/organizations/<org_id>/collections/<coll_id>/users (get_collection_users)
PUT /api/organizations/<org_id>/collections/<coll_id>/users (put_collection_users)
PUT /api/organizations/<org_id> (put_organization)
POST /api/organizations/<org_id> (post_organization)
POST /api/organizations/<org_id>/collections (post_organization_collections)
DELETE /api/organizations/<org_id>/collections/<col_id>/user/<org_user_id> (delete_organization_collection_user)
POST /api/organizations/<org_id>/collections/<col_id>/delete-user/<org_user_id> (post_organization_collection_delete_user)
POST /api/organizations/<org_id>/collections/<col_id> (post_organization_collection_update)
PUT /api/organizations/<org_id>/collections/<col_id> (put_organization_collection_update)
DELETE /api/organizations/<org_id>/collections/<col_id> (delete_organization_collection)
POST /api/organizations/<org_id>/collections/<col_id>/delete (post_organization_collection_delete)
GET /api/ciphers/organization-details?<data..> (get_org_details)
GET /api/organizations/<org_id>/users (get_org_users)
POST /api/organizations/<org_id>/users/invite (send_invite)
POST /api/organizations/<org_id>/users/<user_org>/reinvite (reinvite_user)
POST /api/organizations/<org_id>/users/<org_user_id>/confirm (confirm_invite)
POST /api/organizations/<_org_id>/users/<_org_user_id>/accept (accept_invite)
GET /api/organizations/<org_id>/users/<org_user_id> (get_user)
POST /api/organizations/<org_id>/users/<org_user_id> [1] (edit_user)
PUT /api/organizations/<org_id>/users/<org_user_id> (put_organization_user)
DELETE /api/organizations/<org_id>/users/<org_user_id> (delete_user)
POST /api/organizations/<org_id>/users/<org_user_id>/delete (post_delete_user)
POST /api/ciphers/import-organization?<query..> (post_org_import)
GET /api/organizations/<org_id>/policies (list_policies)
GET /api/organizations/<org_id>/policies/token?<token> (list_policies_token)
GET /api/organizations/<org_id>/policies/<pol_type> (get_policy)
PUT /api/organizations/<org_id>/policies/<pol_type> (put_policy)
GET /api/organizations/<org_id>/tax (get_organization_tax)
GET /api/plans (get_plans)
GET /api/plans/sales-tax-rates (get_plans_tax_rates)
POST /api/organizations/<org_id>/import (import)
GET /api/two-factor (get_twofactor)
POST /api/two-factor/get-recover (get_recover)
POST /api/two-factor/recover (recover)
POST /api/two-factor/disable (disable_twofactor)
PUT /api/two-factor/disable (disable_twofactor_put)
POST /api/two-factor/get-authenticator (generate_authenticator)
POST /api/two-factor/authenticator (activate_authenticator)
PUT /api/two-factor/authenticator (activate_authenticator_put)
POST /api/two-factor/get-duo (get_duo)
POST /api/two-factor/duo (activate_duo)
PUT /api/two-factor/duo (activate_duo_put)
POST /api/two-factor/get-email (get_email)
POST /api/two-factor/send-email-login (send_email_login)
POST /api/two-factor/send-email (send_email)
PUT /api/two-factor/email (email)
POST /api/two-factor/get-u2f (generate_u2f)
POST /api/two-factor/get-u2f-challenge (generate_u2f_challenge)
POST /api/two-factor/u2f (activate_u2f)
PUT /api/two-factor/u2f (activate_u2f_put)
DELETE /api/two-factor/u2f (delete_u2f)
POST /api/two-factor/get-yubikey (generate_yubikey)
POST /api/two-factor/yubikey (activate_yubikey)
PUT /api/two-factor/yubikey (activate_yubikey_put)
POST /api/sends (post_send)
POST /api/sends/file multipart/form-data (post_send_file)
POST /api/sends/access/<access_id> (post_access)
POST /api/sends/<send_id>/access/file/<file_id> (post_access_file)
PUT /api/sends/<id> (put_send)
DELETE /api/sends/<id> (delete_send)
PUT /api/sends/<id>/remove-password (put_remove_password)
PUT /api/devices/identifier/<uuid>/clear-token (clear_device_token)
PUT /api/devices/identifier/<uuid>/token (put_device_token)
GET /api/settings/domains (get_eq_domains)
POST /api/settings/domains (post_eq_domains)
PUT /api/settings/domains (put_eq_domains)
GET /api/hibp/breach?<username> (hibp_breach)
GET /admin (admin_disabled)
POST /identity/connect/token (login)
GET /icons/<domain>/icon.png (icon)
POST /notifications/hub/negotiate (negotiate)
GET /notifications/hub (websockets_err)

View File

@ -0,0 +1,25 @@
GET / (index)
GET /<category> (category)
GET /governance (governance)
GET /governance/<section>/<team> [2] (team)
GET /production/users (production)
GET /sponsors (sponsors)
GET /<category>/<subject> [4] (subject)
GET /static/<file..> (files)
GET /robots.txt (robots_txt)
GET /logos/<file..> (logos)
GET /components/<_file..> (components)
GET /<locale> [3] (index_locale)
GET /<locale>/<category> [11] (category_locale)
GET /<locale>/governance [10] (governance_locale)
GET /<locale>/governance/<section>/<team> [12] (team_locale)
GET /<locale>/production/users [10] (production_locale)
GET /<locale>/sponsors [10] (sponsors_locale)
GET /<locale>/<category>/<subject> [14] (subject_locale)
GET /<locale>/components/<_file..> [12] (components_locale)
GET /<dest> [19] (redirect)
GET /pdfs/<dest> (redirect_pdfs)
GET /en-US (redirect_bare_en_us)
GET /<_locale> [20] (redirect_bare_locale)
GET /en-US/<dest> (redirect_en_us)
GET /<_locale>/<dest> [20] (redirect_locale)

View File

@ -356,6 +356,12 @@ impl FromStr for ContentType {
}
}
impl From<MediaType> for ContentType {
fn from(media_type: MediaType) -> Self {
ContentType(media_type)
}
}
impl fmt::Display for ContentType {
/// Formats the ContentType as an HTTP Content-Type value.
///

View File

@ -72,18 +72,5 @@ yansi = "0.5"
version_check = "0.9.1"
[dev-dependencies]
bencher = "0.1"
figment = { version = "0.10", features = ["test"] }
pretty_assertions = "0.7"
[[bench]]
name = "format-routing"
harness = false
[[bench]]
name = "ranked-routing"
harness = false
[[bench]]
name = "simple-routing"
harness = false

View File

@ -1,55 +0,0 @@
#[macro_use] extern crate rocket;
#[macro_use] extern crate bencher;
use rocket::local::blocking::Client;
#[get("/", format = "application/json")]
fn get() -> &'static str { "get" }
#[post("/", format = "application/json")]
fn post() -> &'static str { "post" }
fn rocket() -> rocket::Rocket {
let config = rocket::Config {
log_level: rocket::config::LogLevel::Off,
..rocket::Config::debug_default()
};
rocket::custom(config).mount("/", routes![get, post])
}
use bencher::Bencher;
use rocket::http::{Accept, ContentType};
fn accept_format(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
let request = client.get("/").header(Accept::JSON);
b.iter(|| { request.clone().dispatch(); });
}
fn wrong_accept_format(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
let request = client.get("/").header(Accept::HTML);
b.iter(|| { request.clone().dispatch(); });
}
fn content_type_format(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
let request = client.post("/").header(ContentType::JSON);
b.iter(|| { request.clone().dispatch(); });
}
fn wrong_content_type_format(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
let request = client.post("/").header(ContentType::Plain);
b.iter(|| { request.clone().dispatch(); });
}
benchmark_main!(benches);
benchmark_group! {
benches,
accept_format,
wrong_accept_format,
content_type_format,
wrong_content_type_format,
}

View File

@ -1,72 +0,0 @@
#[macro_use] extern crate rocket;
#[macro_use] extern crate bencher;
#[get("/", format = "application/json", rank = 1)]
fn get() -> &'static str { "json" }
#[get("/", format = "text/html", rank = 2)]
fn get2() -> &'static str { "html" }
#[get("/", format = "text/plain", rank = 3)]
fn get3() -> &'static str { "plain" }
#[post("/", format = "application/json")]
fn post() -> &'static str { "json" }
#[post("/", format = "text/html")]
fn post2() -> &'static str { "html" }
#[post("/", format = "text/plain")]
fn post3() -> &'static str { "plain" }
fn rocket() -> rocket::Rocket {
let config = rocket::Config {
log_level: rocket::config::LogLevel::Off,
..rocket::Config::debug_default()
};
rocket::custom(config)
.mount("/", routes![get, get2, get3])
.mount("/", routes![post, post2, post3])
}
use bencher::Bencher;
use rocket::local::blocking::Client;
use rocket::http::{Accept, ContentType};
fn accept_format(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
let requests = vec![
client.get("/").header(Accept::JSON),
client.get("/").header(Accept::HTML),
client.get("/").header(Accept::Plain),
];
b.iter(|| {
for request in &requests {
request.clone().dispatch();
}
});
}
fn content_type_format(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
let requests = vec![
client.post("/").header(ContentType::JSON),
client.post("/").header(ContentType::HTML),
client.post("/").header(ContentType::Plain),
];
b.iter(|| {
for request in &requests {
request.clone().dispatch();
}
});
}
benchmark_main!(benches);
benchmark_group! {
benches,
accept_format,
content_type_format,
}

View File

@ -1,122 +0,0 @@
#[macro_use] extern crate rocket;
#[macro_use] extern crate bencher;
#[get("/")]
fn hello_world() -> &'static str { "Hello, world!" }
#[put("/")]
fn put_index() -> &'static str { "index" }
#[post("/")]
fn post_index() -> &'static str { "index" }
#[get("/a")]
fn index_a() -> &'static str { "index" }
#[get("/b")]
fn index_b() -> &'static str { "index" }
#[get("/c")]
fn index_c() -> &'static str { "index" }
#[get("/<_a>")]
fn index_dyn_a(_a: &str) -> &'static str { "index" }
fn hello_world_rocket() -> rocket::Rocket {
let config = rocket::Config {
log_level: rocket::config::LogLevel::Off,
..rocket::Config::debug_default()
};
rocket::custom(config).mount("/", routes![hello_world])
}
fn rocket() -> rocket::Rocket {
hello_world_rocket()
.mount("/", routes![
put_index, post_index, index_a, index_b, index_c, index_dyn_a
])
}
use bencher::Bencher;
use rocket::local::blocking::Client;
fn bench_hello_world(b: &mut Bencher) {
let client = Client::tracked(hello_world_rocket()).unwrap();
b.iter(|| {
client.get("/").dispatch();
});
}
fn bench_single_get_index(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
b.iter(|| {
client.get("/").dispatch();
});
}
fn bench_get_put_post_index(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
// Hold all of the requests we're going to make during the benchmark.
let mut requests = vec![];
requests.push(client.get("/"));
requests.push(client.put("/"));
requests.push(client.post("/"));
b.iter(|| {
for request in &requests {
request.clone().dispatch();
}
});
}
fn bench_dynamic(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
// Hold all of the requests we're going to make during the benchmark.
let mut requests = vec![];
requests.push(client.get("/abc"));
requests.push(client.get("/abcdefg"));
requests.push(client.get("/123"));
b.iter(|| {
for request in &requests {
request.clone().dispatch();
}
});
}
fn bench_simple_routing(b: &mut Bencher) {
let client = Client::tracked(rocket()).unwrap();
// Hold all of the requests we're going to make during the benchmark.
let mut requests = vec![];
for route in client.rocket().routes() {
let request = client.req(route.method, route.uri.path());
requests.push(request);
}
// A few more for the dynamic route.
requests.push(client.get("/abc"));
requests.push(client.get("/abcdefg"));
requests.push(client.get("/123"));
b.iter(|| {
for request in &requests {
request.clone().dispatch();
}
});
}
benchmark_main!(benches);
benchmark_group! {
benches,
bench_hello_world,
bench_single_get_index,
bench_get_put_post_index,
bench_dynamic,
bench_simple_routing,
}