countrylist <- list(
  "ad" = "Andorra",
  "ae" = "the United Arab Emirates",
  "af" = "Afghanistan",
  "ag" = "Antigua and Barbuda",
  "ai" = "Anguilla",
  "al" = "Albania",
  "am" = "Armenia",
  "an" = "the Netherlands Antilles",
  "ao" = "Angola",
  "aq" = "Antarctica",
  "ar" = "Argentina",
  "as" = "American Samoa",
  "at" = "Austria",
  "au" = "Australia",
  "aw" = "Aruba",
  "ax" = "the Aland Islands",
  "az" = "Azerbaijan",
  "ba" = "Bosnia and Herzegovina",
  "bb" = "Barbados",
  "bd" = "Bangladesh",
  "be" = "Belgium",
  "bf" = "Burkina Faso",
  "bg" = "Bulgaria",
  "bh" = "Bahrain",
  "bi" = "Burundi",
  "bj" = "Benin",
  "bl" = "Saint Bartelemey",
  "bm" = "Bermuda",
  "bn" = "Brunei",
  "bo" = "Bolivia",
  "bq" = "Bonaire, Sint Eustatius and Saba",
  "br" = "Brazil",
  "bs" = "the Bahamas",
  "bt" = "Bhutan",
  "bv" = "the Bouvet Island",
  "bw" = "Botswana",
  "by" = "Belarus",
  "bz" = "Belize",
  "ca" = "Canada",
  "cc" = "the Cocos (Keeling) Islands",
  "cd" = "the Democratic Republic of the Congo",
  "cf" = "Central African Republic",
  "cg" = "Congo",
  "ch" = "Switzerland",
  "ci" = "Côte d'Ivoire",
  "ck" = "the Cook Islands",
  "cl" = "Chile",
  "cm" = "Cameroon",
  "cn" = "China",
  "co" = "Colombia",
  "cr" = "Costa Rica",
  "cu" = "Cuba",
  "cv" = "Cape Verde",
  "cw" = "Curaçao",
  "cx" = "the Christmas Island",
  "cy" = "Cyprus",
  "cz" = "the Czech Republic",
  "de" = "Germany",
  "dj" = "Djibouti",
  "dk" = "Denmark",
  "dm" = "Dominica",
  "do" = "the Dominican Republic",
  "dz" = "Algeria",
  "ec" = "Ecuador",
  "ee" = "Estonia",
  "eg" = "Egypt",
  "eh" = "the Western Sahara",
  "er" = "Eritrea",
  "es" = "Spain",
  "et" = "Ethiopia",
  "fi" = "Finland",
  "fj" = "Fiji",
  "fk" = "the Falkland Islands (Malvinas)",
  "fm" = "the Federated States of Micronesia",
  "fo" = "the Faroe Islands",
  "fr" = "France",
  "ga" = "Gabon",
  "gb" = "the United Kingdom",
  "gd" = "Grenada",
  "ge" = "Georgia",
  "gf" = "French Guiana",
  "gg" = "Guernsey",
  "gh" = "Ghana",
  "gi" = "Gibraltar",
  "gl" = "Greenland",
  "gm" = "Gambia",
  "gn" = "Guinea",
  "gp" = "Guadeloupe",
  "gq" = "Equatorial Guinea",
  "gr" = "Greece",
  "gs" = "South Georgia and the South Sandwich Islands",
  "gt" = "Guatemala",
  "gu" = "Guam",
  "gw" = "Guinea-Bissau",
  "gy" = "Guyana",
  "hk" = "Hong Kong",
  "hm" = "Heard Island and McDonald Islands",
  "hn" = "Honduras",
  "hr" = "Croatia",
  "ht" = "Haiti",
  "hu" = "Hungary",
  "id" = "Indonesia",
  "ie" = "Ireland",
  "il" = "Israel",
  "im" = "the Isle of Man",
  "in" = "India",
  "io" = "the British Indian Ocean Territory",
  "iq" = "Iraq",
  "ir" = "Iran",
  "is" = "Iceland",
  "it" = "Italy",
  "je" = "Jersey",
  "jm" = "Jamaica",
  "jo" = "Jordan",
  "jp" = "Japan",
  "ke" = "Kenya",
  "kg" = "Kyrgyzstan",
  "kh" = "Cambodia",
  "ki" = "Kiribati",
  "km" = "Comoros",
  "kn" = "Saint Kitts and Nevis",
  "kp" = "North Korea",
  "kr" = "the Republic of Korea",
  "kw" = "Kuwait",
  "ky" = "the Cayman Islands",
  "kz" = "Kazakhstan",
  "la" = "Laos",
  "lb" = "Lebanon",
  "lc" = "Saint Lucia",
  "li" = "Liechtenstein",
  "lk" = "Sri Lanka",
  "lr" = "Liberia",
  "ls" = "Lesotho",
  "lt" = "Lithuania",
  "lu" = "Luxembourg",
  "lv" = "Latvia",
  "ly" = "Libya",
  "ma" = "Morocco",
  "mc" = "Monaco",
  "md" = "the Republic of Moldova",
  "me" = "Montenegro",
  "mf" = "Saint Martin",
  "mg" = "Madagascar",
  "mh" = "the Marshall Islands",
  "mk" = "Macedonia",
  "ml" = "Mali",
  "mm" = "Burma",
  "mn" = "Mongolia",
  "mo" = "Macau",
  "mp" = "the Northern Mariana Islands",
  "mq" = "Martinique",
  "mr" = "Mauritania",
  "ms" = "Montserrat",
  "mt" = "Malta",
  "mu" = "Mauritius",
  "mv" = "the Maldives",
  "mw" = "Malawi",
  "mx" = "Mexico",
  "my" = "Malaysia",
  "mz" = "Mozambique",
  "na" = "Namibia",
  "nc" = "New Caledonia",
  "ne" = "Niger",
  "nf" = "Norfolk Island",
  "ng" = "Nigeria",
  "ni" = "Nicaragua",
  "nl" = "the Netherlands",
  "no" = "Norway",
  "np" = "Nepal",
  "nr" = "Nauru",
  "nu" = "Niue",
  "nz" = "New Zealand",
  "om" = "Oman",
  "pa" = "Panama",
  "pe" = "Peru",
  "pf" = "French Polynesia",
  "pg" = "Papua New Guinea",
  "ph" = "the Philippines",
  "pk" = "Pakistan",
  "pl" = "Poland",
  "pm" = "Saint Pierre and Miquelon",
  "pn" = "the Pitcairn Islands",
  "pr" = "Puerto Rico",
  "ps" = "the Palestinian Territory",
  "pt" = "Portugal",
  "pw" = "Palau",
  "py" = "Paraguay",
  "qa" = "Qatar",
  "re" = "Reunion",
  "ro" = "Romania",
  "rs" = "Serbia",
  "ru" = "Russia",
  "rw" = "Rwanda",
  "sa" = "Saudi Arabia",
  "sb" = "the Solomon Islands",
  "sc" = "the Seychelles",
  "sd" = "Sudan",
  "se" = "Sweden",
  "sg" = "Singapore",
  "sh" = "Saint Helena",
  "si" = "Slovenia",
  "sj" = "Svalbard and Jan Mayen",
  "sk" = "Slovakia",
  "sl" = "Sierra Leone",
  "sm" = "San Marino",
  "sn" = "Senegal",
  "so" = "Somalia",
  "sr" = "Suriname",
  "ss" = "South Sudan",
  "st" = "São Tomé and Príncipe",
  "sv" = "El Salvador",
  "sx" = "Sint Maarten",
  "sy" = "the Syrian Arab Republic",
  "sz" = "Swaziland",
  "tc" = "Turks and Caicos Islands",
  "td" = "Chad",
  "tf" = "the French Southern Territories",
  "tg" = "Togo",
  "th" = "Thailand",
  "tj" = "Tajikistan",
  "tk" = "Tokelau",
  "tl" = "East Timor",
  "tm" = "Turkmenistan",
  "tn" = "Tunisia",
  "to" = "Tonga",
  "tr" = "Turkey",
  "tt" = "Trinidad and Tobago",
  "tv" = "Tuvalu",
  "tw" = "Taiwan",
  "tz" = "the United Republic of Tanzania",
  "ua" = "Ukraine",
  "ug" = "Uganda",
  "um" = "the United States Minor Outlying Islands",
  "us" = "the United States",
  "uy" = "Uruguay",
  "uz" = "Uzbekistan",
  "va" = "Vatican City",
  "vc" = "Saint Vincent and the Grenadines",
  "ve" = "Venezuela",
  "vg" = "the British Virgin Islands",
  "vi" = "the United States Virgin Islands",
  "vn" = "Vietnam",
  "vu" = "Vanuatu",
  "wf" = "Wallis and Futuna",
  "ws" = "Samoa",
  "xk" = "Kosovo",
  "ye" = "Yemen",
  "yt" = "Mayotte",
  "za" = "South Africa",
  "zm" = "Zambia",
  "zw" = "Zimbabwe")

countryname <- function(country) {
  res <- countrylist[[country]]
  if (is.null(res))
    res <- "no-man's-land"
  res
}

# Helper function that takes date limits as input and returns major breaks as
# output. The main difference to the built-in major breaks is that we're trying
# harder to align major breaks with first days of weeks (Sundays), months,
# quarters, or years.
custom_breaks <- function(input) {
  scales_index <- cut(as.numeric(max(input) - min(input)),
    c(-1, 7, 12, 56, 180, 600, 2000, Inf), labels = FALSE)
  from_print_format <- c("%F", "%F", "%Y-W%U-7", "%Y-%m-01", "%Y-01-01",
    "%Y-01-01", "%Y-01-01")[scales_index]
  from_parse_format <- ifelse(scales_index == 3, "%Y-W%U-%u", "%F")
  by <- c("1 day", "2 days", "1 week", "1 month", "3 months", "1 year",
    "2 years")[scales_index]
  seq(as.Date(as.character(min(input), from_print_format),
    format = from_parse_format), max(input), by = by)
}

# Helper function that takes date limits as input and returns minor breaks as
# output. As opposed to the built-in minor breaks, we're not just adding one
# minor break half way through between two major breaks. Instead, we're plotting
# a minor break for every day, week, month, or quarter between two major breaks.
custom_minor_breaks <- function(input) {
  scales_index <- cut(as.numeric(max(input) - min(input)),
    c(-1, 7, 12, 56, 180, 600, 2000, Inf), labels = FALSE)
  from_print_format <- c("%F", "%F", "%F", "%Y-W%U-7", "%Y-%m-01", "%Y-01-01",
    "%Y-01-01")[scales_index]
  from_parse_format <- ifelse(scales_index == 4, "%Y-W%U-%u", "%F")
  by <- c("1 day", "1 day", "1 day", "1 week", "1 month", "3 months",
    "1 year")[scales_index]
  seq(as.Date(as.character(min(input), from_print_format),
    format = from_parse_format), max(input), by = by)
}

# Helper function that takes breaks as input and returns labels as output. We're
# going all ISO-8601 here, though we're not just writing %Y-%m-%d everywhere,
# but %Y-%m or %Y if all breaks are on the first of a month or even year.
custom_labels <- function(breaks) {
  if (all(format(breaks, format = "%m-%d") == "01-01", na.rm = TRUE)) {
    format(breaks, format = "%Y")
  } else {
    if (all(format(breaks, format = "%d") == "01", na.rm = TRUE)) {
      format(breaks, format = "%Y-%m")
    } else {
      format(breaks, format = "%F")
    }
  }
}

# Helper function to format numbers in non-scientific notation with spaces as
# thousands separator.
formatter <- function(x, ...) {
  format(x, ..., scientific = FALSE, big.mark = " ")
}

theme_update(
  # Make plot title centered, and leave some room to the plot.
  plot.title = element_text(hjust = 0.5, margin = margin(b = 11)),

  # Leave a little more room to the right for long x axis labels.
  plot.margin = margin(5.5, 11, 5.5, 5.5)
)

# Set the default line size of geom_line() to 1.
update_geom_defaults("line", list(size = 1))

copyright_notice <- "The Tor Project - https://metrics.torproject.org/"

stats_dir <- "/srv/metrics.torproject.org/metrics/shared/stats/"

rdata_dir <- "/srv/metrics.torproject.org/metrics/shared/RData/"

# Helper function that copies the appropriate no data object to filename.
copy_no_data <- function(filename) {
  len <- nchar(filename)
  extension <- substr(filename, len - 3, len)
  if (".csv" == extension) {
    write("# No data available for the given parameters.", file=filename)
  } else {
    file.copy(paste(rdata_dir, "no-data-available", extension, sep = ""),
      filename)
  }
}

# Helper function wrapping calls into error handling.
robust_call <- function(wrappee, filename) {
  tryCatch(eval(wrappee), error = function(e) copy_no_data(filename),
     finally = if (!file.exists(filename) || file.size(filename) == 0) {
       copy_no_data(filename)
       })
}

prepare_networksize <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "servers.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(flag == "") %>%
    filter(country == "") %>%
    filter(version == "") %>%
    filter(platform == "") %>%
    filter(ec2bridge == "") %>%
    select(date, relays, bridges)
}

plot_networksize <- function(start_p, end_p, path_p) {
  prepare_networksize(start_p, end_p) %>%
    gather(variable, value, -date) %>%
    complete(date = full_seq(date, period = 1),
      variable = c("relays", "bridges")) %>%
    ggplot(aes(x = date, y = value, colour = variable)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    scale_colour_hue("", breaks = c("relays", "bridges"),
        labels = c("Relays", "Bridges")) +
    ggtitle("Number of relays") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_networksize <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_networksize(start_p, end_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_versions <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "servers.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(flag == "") %>%
    filter(country == "") %>%
    filter(version != "") %>%
    filter(platform == "") %>%
    filter(ec2bridge == "") %>%
    select(date, version, relays)
}

plot_versions <- function(start_p, end_p, path_p) {
  s <- prepare_versions(start_p, end_p)
  known_versions <- c("Other", "0.1.0", "0.1.1", "0.1.2", "0.2.0",
        "0.2.1", "0.2.2", "0.2.3", "0.2.4", "0.2.5", "0.2.6", "0.2.7",
        "0.2.8", "0.2.9", "0.3.0", "0.3.1", "0.3.2", "0.3.3", "0.3.4")
  getPalette <- colorRampPalette(brewer.pal(12, "Paired"))
  colours <- data.frame(breaks = known_versions,
    values = rep(brewer.pal(min(12, length(known_versions)), "Paired"),
                 len = length(known_versions)),
    stringsAsFactors = FALSE)
  versions <- s[s$version %in% known_versions, ]
  visible_versions <- sort(unique(versions$version))
  ggplot(versions, aes(x = date, y = relays, colour = version)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    scale_colour_manual(name = "Tor version",
      values = colours[colours$breaks %in% visible_versions, 2],
      breaks = visible_versions) +
    ggtitle("Relay versions") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_versions <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_versions(start_p, end_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_platforms <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "servers.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(flag == "") %>%
    filter(country == "") %>%
    filter(version == "") %>%
    filter(platform != "") %>%
    filter(ec2bridge == "") %>%
    select(date, platform, relays) %>%
    mutate(platform = ifelse(platform == "Darwin", "macOS",
      as.character(platform)))
}

plot_platforms <- function(start_p, end_p, path_p) {
  prepare_platforms(start_p, end_p) %>%
    ggplot(aes(x = date, y = relays, colour = platform)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    scale_colour_manual(name = "Platform",
      breaks = c("Linux", "macOS", "BSD", "Windows", "Other"),
      values = c("#E69F00", "#56B4E9", "#009E73", "#0072B2", "#333333")) +
    ggtitle("Relay platforms") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_platforms <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_platforms(start_p, end_p) %>%
    mutate(platform = tolower(platform)) %>%
    spread(platform, relays) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_bandwidth <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(isexit != "") %>%
    filter(isguard != "") %>%
    group_by(date) %>%
    summarize(advbw = sum(advbw) * 8 / 1e9,
      bwhist = sum(bwread + bwwrite) * 8 / 2e9) %>%
    select(date, advbw, bwhist)
}

plot_bandwidth <- function(start_p, end_p, path_p) {
  prepare_bandwidth(start_p, end_p) %>%
    gather(variable, value, -date) %>%
    ggplot(aes(x = date, y = value, colour = variable)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
      limits = c(0, NA)) +
    scale_colour_hue(name = "", h.start = 90,
        breaks = c("advbw", "bwhist"),
        labels = c("Advertised bandwidth", "Bandwidth history")) +
    ggtitle("Total relay bandwidth") +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_bandwidth <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_bandwidth(start_p, end_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_bwhist_flags <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(isexit != "") %>%
    filter(isguard != "") %>%
    mutate(variable = ifelse(isexit == "t",
        ifelse(isguard == "t", "guard_and_exit", "exit_only"),
        ifelse(isguard == "t", "guard_only", "middle_only")),
      value = (bwread + bwwrite) * 8 / 2e9) %>%
    select(date, variable, value)
}

plot_bwhist_flags <- function(start_p, end_p, path_p) {
  prepare_bwhist_flags(start_p, end_p) %>%
    complete(date = full_seq(date, period = 1),
      variable = unique(variable)) %>%
    ggplot(aes(x = date, y = value, colour = variable)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
      limits = c(0, NA)) +
    scale_colour_manual(name = "",
      breaks = c("exit_only", "guard_and_exit", "guard_only", "middle_only"),
      labels = c("Exit only", "Guard & Exit", "Guard only", "Middle only"),
      values = c("#E69F00", "#56B4E9", "#009E73", "#0072B2")) +
    ggtitle("Bandwidth history by relay flags") +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_bwhist_flags <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_bwhist_flags(start_p, end_p) %>%
    spread(variable, value) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_dirbytes <- function(start_p, end_p, path_p) {
  read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(isexit == "") %>%
    filter(isguard == "") %>%
    mutate(dirread = dirread * 8 / 1e9,
      dirwrite = dirwrite * 8 / 1e9) %>%
    select(date, dirread, dirwrite)
}

plot_dirbytes <- function(start_p, end_p, path_p) {
  prepare_dirbytes(start_p, end_p) %>%
    gather(variable, value, -date) %>%
    ggplot(aes(x = date, y = value, colour = variable)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
      limits = c(0, NA)) +
    scale_colour_hue(name = "",
        breaks = c("dirwrite", "dirread"),
        labels = c("Written dir bytes", "Read dir bytes")) +
    ggtitle("Number of bytes spent on answering directory requests") +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_dirbytes <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_dirbytes(start_p, end_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_relayflags <- function(start_p, end_p, flag_p) {
  read.csv(paste(stats_dir, "servers.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(country == "") %>%
    filter(version == "") %>%
    filter(platform == "") %>%
    filter(ec2bridge == "") %>%
    mutate(flag = ifelse(flag == "", "Running", as.character(flag))) %>%
    filter(if (!is.null(flag_p)) flag %in% flag_p else TRUE) %>%
    select(date, flag, relays)
}

plot_relayflags <- function(start_p, end_p, flag_p, path_p) {
  prepare_relayflags(start_p, end_p, flag_p) %>%
    complete(date = full_seq(date, period = 1), flag = unique(flag)) %>%
    ggplot(aes(x = date, y = relays, colour = as.factor(flag))) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    scale_colour_manual(name = "Relay flags", values = c("#E69F00",
        "#56B4E9", "#009E73", "#EE6A50", "#000000", "#0072B2"),
        breaks = flag_p, labels = flag_p) +
    ggtitle("Number of relays with relay flags assigned") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_relayflags <- function(start_p = NULL, end_p = NULL, flag_p = NULL,
    path_p) {
  prepare_relayflags(start_p, end_p, flag_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

plot_torperf <- function(start_p, end_p, source_p, server_p, filesize_p,
    path_p) {
  filesize_val <- ifelse(filesize_p == "50kb", 50 * 1024,
          ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024))
  t <- read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""),
    colClasses = c("date" = "Date", "source" = "character"))
  known_sources <- c("all", unique(t[t$source != "", "source"]))
  colours <- data.frame(source = known_sources,
      colour = brewer.pal(length(known_sources), "Paired"),
      stringsAsFactors = FALSE)
  colour <- colours[colours$source == source_p, "colour"]
  filesizes <- data.frame(filesizes = c("5mb", "1mb", "50kb"),
      label = c("5 MiB", "1 MiB", "50 KiB"), stringsAsFactors = FALSE)
  filesize_str <- filesizes[filesizes$filesize == filesize_p, "label"]
  t[t$date >= as.Date(start_p) & t$date <= as.Date(end_p) &
         t$filesize == filesize_val &
         t$source == ifelse(source_p == "all", "", source_p) &
         t$server == server_p, ] %>%
    transmute(date, q1 = q1 / 1e3, md = md / 1e3, q3 = q3 / 1e3) %>%
    complete(date = full_seq(date, period = 1)) %>%
    ggplot(aes(x = date, y = md, fill = "line")) +
    geom_line(colour = colour, size = 0.75) +
    geom_ribbon(aes(x = date, ymin = q1, ymax = q3, fill = "ribbon")) +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "s"),
      limits = c(0, NA)) +
    scale_fill_manual(name = paste("Measured times on",
        ifelse(source_p == "all", "all sources", source_p), "per day"),
      breaks = c("line", "ribbon"),
      labels = c("Median", "1st to 3rd quartile"),
      values = paste(colour, c("", "66"), sep = "")) +
    ggtitle(paste("Time to complete", filesize_str,
        "request to", server_p, "server")) +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

# Ideally, this function would share code with plot_torperf by using a
# common prepare_torperf function. This just turned out to be a bit
# harder than for other functions, because plot_torperf uses different
# colours based on which sources exist, unrelated to which source is
# plotted. Left as future work.
write_torperf <- function(start_p = NULL, end_p = NULL, source_p = NULL,
    server_p = NULL, filesize_p = NULL, path_p) {
  read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(source_p))
        source == ifelse(source_p == "all", "", source_p) else TRUE) %>%
    filter(if (!is.null(server_p)) server == server_p else TRUE) %>%
    filter(if (!is.null(filesize_p))
        filesize == ifelse(filesize_p == "50kb", 50 * 1024,
        ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else
        TRUE) %>%
    transmute(date, filesize, source, server, q1 = q1 / 1e3, md = md / 1e3,
      q3 = q3 / 1e3) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_torperf_failures <- function(start_p, end_p, source_p, server_p,
    filesize_p) {
  read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(filesize_p))
        filesize == ifelse(filesize_p == "50kb", 50 * 1024,
        ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else
        TRUE) %>%
    filter(if (!is.null(source_p))
        source == ifelse(source_p == "all", "", source_p) else TRUE) %>%
    filter(if (!is.null(server_p)) server == server_p else TRUE) %>%
    filter(requests > 0) %>%
    transmute(date, filesize, source, server, timeouts = timeouts / requests,
        failures = failures / requests)
}

plot_torperf_failures <- function(start_p, end_p, source_p, server_p,
    filesize_p, path_p) {
  filesizes <- data.frame(filesizes = c("5mb", "1mb", "50kb"),
      label = c("5 MiB", "1 MiB", "50 KiB"), stringsAsFactors = FALSE)
  filesize_str <- filesizes[filesizes$filesize == filesize_p, "label"]
  prepare_torperf_failures(start_p, end_p, source_p, server_p, filesize_p) %>%
    gather(variable, value, -c(date, filesize, source, server)) %>%
    ggplot(aes(x = date, y = value, colour = variable)) +
    geom_point(size = 2) +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) +
    scale_colour_hue(name = paste("Problems encountered on",
        ifelse(source_p == "all", "all sources", source_p)),
        h.start = 45, breaks = c("timeouts", "failures"),
        labels = c("Timeouts", "Failures")) +
    ggtitle(paste("Timeouts and failures of", filesize_str,
        "requests to", server_p, "server")) +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_torperf_failures <- function(start_p = NULL, end_p = NULL,
    source_p = NULL, server_p = NULL, filesize_p = NULL, path_p) {
  prepare_torperf_failures(start_p, end_p, source_p, server_p, filesize_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_onionperf_buildtimes <- function(start_p, end_p, source_p) {
    read.csv(paste(stats_dir, "buildtimes.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(source_p))
        source == ifelse(source_p == "all", "", source_p) else TRUE)
}

write_onionperf_buildtimes <- function(start_p = NULL, end_p = NULL,
    source_p = NULL, path_p) {
  prepare_onionperf_buildtimes(start_p, end_p, source_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

plot_onionperf_buildtimes <- function(start_p, end_p, source_p, path_p) {
  prepare_onionperf_buildtimes(start_p, end_p, source_p) %>%
    mutate(date = as.Date(date),
      position = factor(position, levels = seq(1, 3, 1),
        labels = c("1st hop", "2nd hop", "3rd hop"))) %>%
    ggplot(aes(x = date, y = md, colour = position, fill = position)) +
    geom_line(size = 0.75) +
    geom_ribbon(aes(x = as.Date(date), ymin = q1, ymax = q3, alpha = 0.5),
      show.legend = FALSE) +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "ms"),
      limits = c(0, NA)) +
    scale_colour_hue(name = "Medians and interquartile ranges") +
    scale_fill_hue(name = "Medians and interquartile ranges") +
    ggtitle(ifelse(source_p == "all", "Circuit build times on all sources",
        paste("Circuit build times on", source_p))) +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

prepare_onionperf_latencies <- function(start_p, end_p, source_p) {
    read.csv(paste(stats_dir, "latencies.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(source_p))
        source == ifelse(source_p == "all", "", source_p) else TRUE)
}

write_onionperf_latencies <- function(start_p = NULL, end_p = NULL,
    source_p = NULL, path_p) {
  prepare_onionperf_latencies(start_p, end_p, source_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

plot_onionperf_latencies <- function(start_p, end_p, source_p, path_p) {
  prepare_onionperf_latencies(start_p, end_p, source_p) %>%
    mutate(date = as.Date(date),
      server = factor(server, levels = c("public", "onion"),
        labels = c("public server", "onion server"))) %>%
    ggplot(aes(x = date, y = md, colour = server, fill = server)) +
    geom_line(size = 0.75) +
    geom_ribbon(aes(x = as.Date(date), ymin = q1, ymax = q3, alpha = 0.5),
      show.legend = FALSE) +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "ms"),
      limits = c(0, NA)) +
    scale_colour_hue(name = "Medians and interquartile ranges") +
    scale_fill_hue(name = "Medians and interquartile ranges") +
    ggtitle(ifelse(source_p == "all",
        "Circuit round-trip latencies on all sources",
        paste("Circuit round-trip latencies on", source_p))) +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

prepare_connbidirect <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "connbidirect2.csv", sep = ""),
    colClasses = c("date" = "Date", "direction" = "factor")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    mutate(quantile = paste("X", quantile, sep = ""),
      fraction = fraction / 100) %>%
    spread(quantile, fraction)
}

plot_connbidirect <- function(start_p, end_p, path_p) {
  prepare_connbidirect(start_p, end_p) %>%
    ggplot(aes(x = date, y = X0.5, colour = direction)) +
    geom_line(size = 0.75) +
    geom_ribbon(aes(x = date, ymin = X0.25, ymax = X0.75,
                fill = direction), alpha = 0.5, show.legend = FALSE) +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) +
    scale_colour_hue(name = "Medians and interquartile ranges",
                     breaks = c("both", "write", "read"),
        labels = c("Both reading and writing", "Mostly writing",
                   "Mostly reading")) +
    scale_fill_hue(name = "Medians and interquartile ranges",
                   breaks = c("both", "write", "read"),
        labels = c("Both reading and writing", "Mostly writing",
                   "Mostly reading")) +
    ggtitle("Fraction of connections used uni-/bidirectionally") +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_connbidirect <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_connbidirect(start_p, end_p) %>%
    rename(q1 = X0.25, md = X0.5, q3 = X0.75) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_bandwidth_flags <- function(start_p, end_p) {
  b <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
    colClasses = c("date" = "Date"))
  b <- b %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(isexit != "") %>%
    filter(isguard != "")
  b <- data.frame(date = b$date,
                  isexit = b$isexit == "t", isguard = b$isguard == "t",
                  advbw = b$advbw * 8 / 1e9,
                  bwhist = (b$bwread + b$bwwrite) * 8 / 2e9)
  b <- rbind(
    data.frame(b[b$isguard == TRUE, ], flag = "guard"),
    data.frame(b[b$isexit == TRUE, ], flag = "exit"))
  b <- data.frame(date = b$date, advbw = b$advbw, bwhist = b$bwhist,
                  flag = b$flag)
  b <- aggregate(list(advbw = b$advbw, bwhist = b$bwhist),
                 by = list(date = b$date, flag = b$flag), FUN = sum,
                 na.rm = TRUE, na.action = NULL)
  b <- gather(b, type, value, -c(date, flag))
  bandwidth <- b[b$value > 0, ]
  bandwidth <- data.frame(date = bandwidth$date,
    variable = as.factor(paste(bandwidth$flag, "_", bandwidth$type,
    sep = "")), value = bandwidth$value)
  bandwidth$variable <- factor(bandwidth$variable,
    levels = levels(bandwidth$variable)[c(3, 4, 1, 2)])
  bandwidth
}

plot_bandwidth_flags <- function(start_p, end_p, path_p) {
  prepare_bandwidth_flags(start_p, end_p) %>%
    complete(date = full_seq(date, period = 1),
      variable = unique(variable)) %>%
    ggplot(aes(x = date, y = value, colour = variable)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
      limits = c(0, NA)) +
    scale_colour_manual(name = "",
        breaks = c("guard_advbw", "guard_bwhist", "exit_advbw", "exit_bwhist"),
        labels = c("Guard, advertised bandwidth", "Guard, bandwidth history",
                   "Exit, advertised bandwidth", "Exit, bandwidth history"),
        values = c("#E69F00", "#D6C827", "#009E73", "#00C34F")) +
    ggtitle(paste("Advertised bandwidth and bandwidth history by",
        "relay flags")) +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_bandwidth_flags <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_bandwidth_flags(start_p, end_p) %>%
    spread(variable, value) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

plot_userstats <- function(start_p, end_p, node_p, variable_p, value_p,
    events_p, path_p) {
  load(paste(rdata_dir, "clients-", node_p, ".RData", sep = ""))
  c <- data
  u <- c[c$date >= start_p & c$date <= end_p, c("date", "country", "transport",
      "version", "lower", "upper", "clients")]
  u <- rbind(u, data.frame(date = start_p,
      country = ifelse(variable_p == "country" & value_p != "all", value_p, ""),
      transport = ifelse(variable_p == "transport", value_p, ""),
      version = ifelse(variable_p == "version", value_p, ""),
      lower = 0, upper = 0, clients = 0))
  if (node_p == "relay") {
    if (value_p != "all") {
      u <- u[u$country == value_p, ]
      title <- paste("Directly connecting users from", countryname(value_p))
    } else {
      u <- u[u$country == "", ]
      title <- "Directly connecting users"
    }
    u <- aggregate(list(lower = u$lower, upper = u$upper,
                        users = u$clients),
                   by = list(date = as.Date(u$date, "%Y-%m-%d"),
                             value = u$country),
                   FUN = sum)
  } else if (variable_p == "transport") {
    if ("!<OR>" %in% value_p) {
      n <- u[u$transport != "" & u$transport != "<OR>", ]
      n <- aggregate(list(lower = n$lower, upper = n$upper,
                          clients = n$clients),
                     by = list(date = n$date),
                     FUN = sum)
      u <- rbind(u, data.frame(date = n$date,
                               country = "", transport = "!<OR>",
                               version = "", lower = n$lower,
                               upper = n$upper, clients = n$clients))
    }
    if (length(value_p) > 1) {
      u <- u[u$transport %in% value_p, ]
      u <- aggregate(list(lower = u$lower, upper = u$upper,
                          users = u$clients),
                     by = list(date = as.Date(u$date, "%Y-%m-%d"),
                               value = u$transport),
                     FUN = sum)
      title <- paste("Bridge users by transport")
    } else {
      u <- u[u$transport == value_p, ]
      u <- aggregate(list(lower = u$lower, upper = u$upper,
                          users = u$clients),
                     by = list(date = as.Date(u$date, "%Y-%m-%d"),
                               value = u$transport),
                     FUN = sum)
      title <- paste("Bridge users using",
               ifelse(value_p == "<??>", "unknown pluggable transport(s)",
               ifelse(value_p == "<OR>", "default OR protocol",
               ifelse(value_p == "!<OR>", "any pluggable transport",
               ifelse(value_p == "fte", "FTE",
               ifelse(value_p == "websocket", "Flash proxy/websocket",
               paste("transport", value_p)))))))
    }
  } else if (variable_p == "version") {
    u <- u[u$version == value_p, ]
    title <- paste("Bridge users using IP", value_p, sep = "")
    u <- aggregate(list(lower = u$lower, upper = u$upper,
                        users = u$clients),
                   by = list(date = as.Date(u$date, "%Y-%m-%d"),
                             value = u$version),
                   FUN = sum)
  } else {
    if (value_p != "all") {
      u <- u[u$country == value_p, ]
      title <- paste("Bridge users from", countryname(value_p))
    } else {
      u <- u[u$country == "" & u$transport == "" & u$version == "", ]
      title <- "Bridge users"
    }
    u <- aggregate(list(lower = u$lower, upper = u$upper,
                        users = u$clients),
                   by = list(date = as.Date(u$date, "%Y-%m-%d"),
                             value = u$country),
                   FUN = sum)
  }
  u <- merge(x = u, all.y = TRUE, y = data.frame(expand.grid(
             date = seq(from = as.Date(start_p, "%Y-%m-%d"),
             to = as.Date(end_p, "%Y-%m-%d"), by = "1 day"),
             value = ifelse(value_p == "all", "", value_p))))
  if (length(value_p) > 1) {
    plot <- ggplot(u, aes(x = date, y = users, colour = value))
  } else {
    plot <- ggplot(u, aes(x = date, y = users))
  }
  if (length(na.omit(u$users)) > 0 & events_p != "off" &
      variable_p == "country" & length(value_p) == 1 && value_p != "all") {
    upturns <- u[u$users > u$upper, c("date", "users")]
    downturns <- u[u$users < u$lower, c("date", "users")]
    if (events_p == "on") {
      u[!is.na(u$lower) & u$lower < 0, "lower"] <- 0
      plot <- plot +
        geom_ribbon(data = u, aes(ymin = lower, ymax = upper), fill = "gray")
    }
    if (length(upturns$date) > 0)
      plot <- plot +
          geom_point(data = upturns, aes(x = date, y = users), size = 5,
          colour = "dodgerblue2")
    if (length(downturns$date) > 0)
      plot <- plot +
          geom_point(data = downturns, aes(x = date, y = users), size = 5,
          colour = "firebrick2")
  }
  plot <- plot +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    ggtitle(title) +
    labs(caption = copyright_notice)
  if (length(value_p) > 1) {
    plot <- plot +
      scale_colour_hue(name = "", breaks = value_p,
            labels = ifelse(value_p == "<??>", "Unknown PT",
                     ifelse(value_p == "<OR>", "Default OR protocol",
                     ifelse(value_p == "!<OR>", "Any PT",
                     ifelse(value_p == "fte", "FTE",
                     ifelse(value_p == "websocket", "Flash proxy/websocket",
                     value_p))))))
  }
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

plot_userstats_relay_country <- function(start_p, end_p, country_p, events_p,
    path_p) {
  plot_userstats(start_p, end_p, "relay", "country", country_p, events_p,
    path_p)
}

plot_userstats_bridge_country <- function(start_p, end_p, country_p, path_p) {
  plot_userstats(start_p, end_p, "bridge", "country", country_p, "off", path_p)
}

plot_userstats_bridge_transport <- function(start_p, end_p, transport_p,
    path_p) {
  plot_userstats(start_p, end_p, "bridge", "transport", transport_p, "off",
    path_p)
}

plot_userstats_bridge_version <- function(start_p, end_p, version_p, path_p) {
  plot_userstats(start_p, end_p, "bridge", "version", version_p, "off", path_p)
}

write_userstats_relay_country <- function(start_p = NULL, end_p = NULL,
    country_p = NULL, events_p = NULL, path_p) {
  load(paste(rdata_dir, "clients-relay.RData", sep = ""))
  u <- data %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(country_p))
      country == ifelse(country_p == "all", "", country_p) else TRUE) %>%
    filter(transport == "") %>%
    filter(version == "") %>%
    select(date, country, clients, lower, upper, frac) %>%
    rename(users = clients) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

write_userstats_bridge_country <- function(start_p = NULL, end_p = NULL,
    country_p = NULL, path_p) {
  load(paste(rdata_dir, "clients-bridge.RData", sep = ""))
  data %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(country_p))
      country == ifelse(country_p == "all", "", country_p) else TRUE) %>%
    filter(transport == "") %>%
    filter(version == "") %>%
    select(date, country, clients, frac) %>%
    rename(users = clients) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

write_userstats_bridge_transport <- function(start_p = NULL, end_p = NULL,
    transport_p = NULL, path_p) {
  load(paste(rdata_dir, "clients-bridge.RData", sep = ""))
  u <- data %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(country == "") %>%
    filter(version == "") %>%
    filter(transport != "") %>%
    select(date, transport, clients, frac)
  if (is.null(transport_p) || "!<OR>" %in% transport_p) {
    n <- u %>%
      filter(transport != "<OR>") %>%
      group_by(date, frac) %>%
      summarize(clients = sum(clients))
    u <- rbind(u, data.frame(date = n$date, transport = "!<OR>",
                             clients = n$clients, frac = n$frac))
  }
  u %>%
    filter(if (!is.null(transport_p)) transport %in% transport_p else TRUE) %>%
    group_by(date, transport) %>%
    select(date, transport, clients, frac) %>%
    rename(users = clients) %>%
    arrange(date, transport) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

write_userstats_bridge_version <- function(start_p = NULL, end_p = NULL,
    version_p = NULL, path_p) {
  load(paste(rdata_dir, "clients-bridge.RData", sep = ""))
  data %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(country == "") %>%
    filter(transport == "") %>%
    filter(if (!is.null(version_p)) version == version_p else TRUE) %>%
    select(date, version, clients, frac) %>%
    rename(users = clients) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_userstats_bridge_combined <- function(start_p, end_p, country_p) {
  load(paste(rdata_dir, "userstats-bridge-combined.RData", sep = ""))
  data %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(country_p)) country == country_p else TRUE)
}

plot_userstats_bridge_combined <- function(start_p, end_p, country_p, path_p) {
  if (country_p == "all") {
    plot_userstats_bridge_country(start_p, end_p, country_p, path_p)
  } else {
    top <- 3
    u <- prepare_userstats_bridge_combined(start_p, end_p, country_p)
    a <- aggregate(list(mid = (u$high + u$low) / 2),
                   by = list(transport = u$transport), FUN = sum)
    a <- a[order(a$mid, decreasing = TRUE)[1:top], ]
    u <- u[u$transport %in% a$transport, ]
    title <- paste("Bridge users by transport from ",
                   countryname(country_p), sep = "")
    ggplot(u, aes(x = as.Date(date), ymin = low, ymax = high,
                colour = transport, fill = transport)) +
    geom_ribbon(alpha = 0.5, size = 0.5) +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", limits = c(0, NA), labels = formatter) +
    scale_colour_hue("Top-3 transports") +
    scale_fill_hue("Top-3 transports") +
    ggtitle(title) +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
    ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
  }
}

write_userstats_bridge_combined <- function(start_p = NULL, end_p = NULL,
    country_p = NULL, path_p) {
  if (!is.null(country_p) && country_p == "all") {
    write_userstats_bridge_country(start_p, end_p, country_p, path_p)
  } else {
    prepare_userstats_bridge_combined(start_p, end_p, country_p) %>%
      select(date, country, transport, low, high, frac) %>%
      arrange(date, country, transport) %>%
      write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
  }
}

prepare_advbwdist_perc <- function(start_p, end_p, p_p) {
  read.csv(paste(stats_dir, "advbwdist.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(p_p)) percentile %in% as.numeric(p_p) else
      percentile != "") %>%
    transmute(date, percentile = as.factor(percentile),
      variable = ifelse(isexit != "t", "all", "exits"),
      advbw = advbw * 8 / 1e9)
}

plot_advbwdist_perc <- function(start_p, end_p, p_p, path_p) {
  prepare_advbwdist_perc(start_p, end_p, p_p) %>%
    mutate(variable = ifelse(variable == "all", "All relays",
      "Exits only")) %>%
    ggplot(aes(x = date, y = advbw, colour = percentile)) +
    facet_grid(variable ~ .) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
      limits = c(0, NA)) +
    scale_colour_hue(name = "Percentile") +
    ggtitle("Advertised bandwidth distribution") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_advbwdist_perc <- function(start_p = NULL, end_p = NULL, p_p = NULL,
    path_p) {
  prepare_advbwdist_perc(start_p, end_p, p_p) %>%
    spread(variable, advbw) %>%
    rename(p = percentile) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_advbwdist_relay <- function(start_p, end_p, n_p) {
  read.csv(paste(stats_dir, "advbwdist.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(if (!is.null(n_p)) relay %in% as.numeric(n_p) else
      relay != "") %>%
    transmute(date, relay = as.factor(relay),
      variable = ifelse(isexit != "t", "all", "exits"),
      advbw = advbw * 8 / 1e9)
}

plot_advbwdist_relay <- function(start_p, end_p, n_p, path_p) {
  prepare_advbwdist_relay(start_p, end_p, n_p) %>%
    mutate(variable = ifelse(variable == "all", "All relays",
      "Exits only")) %>%
    ggplot(aes(x = date, y = advbw, colour = relay)) +
    facet_grid(variable ~ .) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
      limits = c(0, NA)) +
    scale_colour_hue(name = "n") +
    ggtitle("Advertised bandwidth of n-th fastest relays") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_advbwdist_relay <- function(start_p = NULL, end_p = NULL, n_p = NULL,
    path_p) {
  prepare_advbwdist_relay(start_p, end_p, n_p) %>%
    spread(variable, advbw) %>%
    rename(n = relay) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_hidserv_dir_onions_seen <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "hidserv.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(type == "dir-onions-seen") %>%
    transmute(date, onions = ifelse(frac >= 0.01, wiqm, NA), frac)
}

plot_hidserv_dir_onions_seen <- function(start_p, end_p, path_p) {
  prepare_hidserv_dir_onions_seen(start_p, end_p) %>%
    ggplot(aes(x = date, y = onions)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", limits = c(0, NA), labels = formatter) +
    ggtitle("Unique .onion addresses") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_hidserv_dir_onions_seen <- function(start_p = NULL, end_p = NULL,
    path_p) {
  prepare_hidserv_dir_onions_seen(start_p, end_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_hidserv_rend_relayed_cells <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "hidserv.csv", sep = ""),
    colClasses = c("date" = "Date")) %>%
    filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
    filter(type == "rend-relayed-cells") %>%
    transmute(date,
      relayed = ifelse(frac >= 0.01, wiqm * 8 * 512 / (86400 * 1e9), NA), frac)
}

plot_hidserv_rend_relayed_cells <- function(start_p, end_p, path_p) {
  prepare_hidserv_rend_relayed_cells(start_p, end_p) %>%
    ggplot(aes(x = date, y = relayed)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
      limits = c(0, NA)) +
    ggtitle("Onion-service traffic") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_hidserv_rend_relayed_cells <- function(start_p = NULL, end_p = NULL,
    path_p) {
  prepare_hidserv_rend_relayed_cells(start_p, end_p) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_webstats_tb <- function(start_p, end_p) {
  load(paste(rdata_dir, "webstats-tb.RData", sep = ""))
  data %>%
    filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
    mutate(request_type = factor(request_type))
}

plot_webstats_tb <- function(start_p, end_p, path_p) {
  d <- prepare_webstats_tb(start_p, end_p)
  levels(d$request_type) <- list(
      "Initial downloads" = "tbid",
      "Signature downloads" = "tbsd",
      "Update pings" = "tbup",
      "Update requests" = "tbur")
  ggplot(d, aes(x = log_date, y = count)) +
    geom_point() +
    geom_line() +
    facet_grid(request_type ~ ., scales = "free_y") +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)),
          strip.background = element_rect(fill = NA)) +
    ggtitle("Tor Browser downloads and updates") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_webstats_tb <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_webstats_tb(start_p, end_p) %>%
    rename(date = log_date) %>%
    spread(request_type, count) %>%
    rename(initial_downloads = tbid, signature_downloads = tbsd,
      update_pings = tbup, update_requests = tbur) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_webstats_tb_platform <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "webstats.csv", sep = ""),
    colClasses = c("log_date" = "Date")) %>%
    filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
    filter(request_type == "tbid") %>%
    group_by(log_date, platform) %>%
    summarize(count = sum(count))
}

plot_webstats_tb_platform <- function(start_p, end_p, path_p) {
  prepare_webstats_tb_platform(start_p, end_p) %>%
    ggplot(aes(x = log_date, y = count, colour = platform)) +
    geom_point() +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    scale_colour_hue(name = "Platform",
        breaks = c("w", "m", "l", "o", ""),
        labels = c("Windows", "macOS", "Linux", "Other", "Unknown")) +
    theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)),
          strip.background = element_rect(fill = NA)) +
    ggtitle("Tor Browser downloads by platform") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_webstats_tb_platform <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_webstats_tb_platform(start_p, end_p) %>%
    rename(date = log_date) %>%
    spread(platform, count) %>%
    rename(linux = l, macos = m, windows = w) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

plot_webstats_tb_locale <- function(start_p, end_p, path_p) {
  d <- read.csv(paste(stats_dir, "webstats.csv", sep = ""),
    colClasses = c("log_date" = "Date", "locale" = "character"))
  d <- d[d$log_date >= start_p & d$log_date <= end_p &
         d$request_type == "tbid", ]
  e <- d
  e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN = sum)
  e <- e[order(e$count, decreasing = TRUE), ]
  e <- e[1:5, ]
  d <- aggregate(list(count = d$count), by = list(log_date = d$log_date,
    locale = ifelse(d$locale %in% e$locale, d$locale, "(other)")), FUN = sum)
  ggplot(d, aes(x = log_date, y = count, colour = locale)) +
    geom_point() +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    scale_colour_hue(name = "Locale",
        breaks = c(e$locale, "(other)"),
        labels = c(e$locale, "Other")) +
    theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)),
          strip.background = element_rect(fill = NA)) +
    ggtitle("Tor Browser downloads by locale") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

# Ideally, this function would share code with plot_webstats_tb_locale
# by using a common prepare_webstats_tb_locale function. This just
# turned out to be a bit harder than for other functions, because
# plot_webstats_tb_locale needs the preliminary data frame e for its
# breaks and labels. Left as future work.
write_webstats_tb_locale <- function(start_p = NULL, end_p = NULL, path_p) {
  read.csv(paste(stats_dir, "webstats.csv", sep = ""),
    colClasses = c("log_date" = "Date", "locale" = "character")) %>%
    filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
    filter(request_type == "tbid") %>%
    group_by(log_date, locale) %>%
    summarize(initial_downloads = sum(count)) %>%
    rename(date = log_date) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_webstats_tm <- function(start_p, end_p) {
  load(paste(rdata_dir, "webstats-tm.RData", sep = ""))
  data %>%
    filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
    mutate(request_type = factor(request_type))
}

plot_webstats_tm <- function(start_p, end_p, path_p) {
  d <- prepare_webstats_tm(start_p, end_p)
  levels(d$request_type) <- list(
      "Initial downloads" = "tmid",
      "Update pings" = "tmup")
  ggplot(d, aes(x = log_date, y = count)) +
    geom_point() +
    geom_line() +
    facet_grid(request_type ~ ., scales = "free_y") +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)),
          strip.background = element_rect(fill = NA)) +
    ggtitle("Tor Messenger downloads and updates") +
    labs(caption = copyright_notice)
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_webstats_tm <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_webstats_tm(start_p, end_p) %>%
    rename(date = log_date) %>%
    mutate(request_type = factor(request_type, levels = c("tmid", "tmup"))) %>%
    spread(request_type, count, drop = FALSE) %>%
    rename(initial_downloads = tmid, update_pings = tmup) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_relays_ipv6 <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "ipv6servers.csv", sep = ""),
    colClasses = c("valid_after_date" = "Date")) %>%
    filter(if (!is.null(start_p))
        valid_after_date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p))
        valid_after_date <= as.Date(end_p) else TRUE) %>%
    filter(server == "relay") %>%
    group_by(valid_after_date) %>%
    summarize(total = sum(server_count_sum_avg),
      announced = sum(server_count_sum_avg[announced_ipv6 == "t"]),
      reachable = sum(server_count_sum_avg[reachable_ipv6_relay == "t"]),
      exiting = sum(server_count_sum_avg[exiting_ipv6_relay == "t"])) %>%
    complete(valid_after_date = full_seq(valid_after_date, period = 1)) %>%
    gather(total, announced, reachable, exiting, key = "category",
      value = "count")
}

plot_relays_ipv6 <- function(start_p, end_p, path_p) {
  prepare_relays_ipv6(start_p, end_p) %>%
    ggplot(aes(x = valid_after_date, y = count, colour = category)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    scale_colour_hue(name = "", h.start = 90,
      breaks = c("total", "announced", "reachable", "exiting"),
      labels = c("Total (IPv4) OR", "IPv6 announced OR", "IPv6 reachable OR",
        "IPv6 exiting")) +
    ggtitle("Relays by IP version") +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_relays_ipv6 <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_relays_ipv6(start_p, end_p) %>%
    rename(date = valid_after_date) %>%
    spread(category, count) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_bridges_ipv6 <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "ipv6servers.csv", sep = ""),
    colClasses = c("valid_after_date" = "Date")) %>%
    filter(if (!is.null(start_p))
        valid_after_date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p))
        valid_after_date <= as.Date(end_p) else TRUE) %>%
    filter(server == "bridge") %>%
    group_by(valid_after_date) %>%
    summarize(total = sum(server_count_sum_avg),
      announced = sum(server_count_sum_avg[announced_ipv6 == "t"])) %>%
    complete(valid_after_date = full_seq(valid_after_date, period = 1)) %>%
    gather(total, announced, key = "category", value = "count")
}

plot_bridges_ipv6 <- function(start_p, end_p, path_p) {
  prepare_bridges_ipv6(start_p, end_p) %>%
    ggplot(aes(x = valid_after_date, y = count, colour = category)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
    scale_colour_hue(name = "", h.start = 90,
      breaks = c("total", "announced"),
      labels = c("Total (IPv4) OR", "IPv6 announced OR")) +
    ggtitle("Bridges by IP version") +
    labs(caption = copyright_notice) +
    theme(legend.position = "top")
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_bridges_ipv6 <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_bridges_ipv6(start_p, end_p) %>%
    rename(date = valid_after_date) %>%
    spread(category, count) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

prepare_advbw_ipv6 <- function(start_p, end_p) {
  read.csv(paste(stats_dir, "ipv6servers.csv", sep = ""),
    colClasses = c("valid_after_date" = "Date")) %>%
    filter(if (!is.null(start_p))
        valid_after_date >= as.Date(start_p) else TRUE) %>%
    filter(if (!is.null(end_p))
        valid_after_date <= as.Date(end_p) else TRUE) %>%
    filter(server == "relay") %>%
    group_by(valid_after_date) %>%
    summarize(total = sum(advertised_bandwidth_bytes_sum_avg),
      total_guard = sum(advertised_bandwidth_bytes_sum_avg[guard_relay != "f"]),
      total_exit = sum(advertised_bandwidth_bytes_sum_avg[exit_relay != "f"]),
      reachable_guard = sum(advertised_bandwidth_bytes_sum_avg[
        reachable_ipv6_relay != "f" & guard_relay != "f"]),
      reachable_exit = sum(advertised_bandwidth_bytes_sum_avg[
        reachable_ipv6_relay != "f" & exit_relay != "f"]),
      exiting = sum(advertised_bandwidth_bytes_sum_avg[
        exiting_ipv6_relay != "f"])) %>%
    complete(valid_after_date = full_seq(valid_after_date, period = 1)) %>%
    gather(total, total_guard, total_exit, reachable_guard, reachable_exit,
      exiting, key = "category", value = "advbw") %>%
    mutate(advbw = advbw * 8 / 1e9)
}

plot_advbw_ipv6 <- function(start_p, end_p, path_p) {
  prepare_advbw_ipv6(start_p, end_p) %>%
    ggplot(aes(x = valid_after_date, y = advbw, colour = category)) +
    geom_line() +
    scale_x_date(name = "", breaks = custom_breaks,
      labels = custom_labels, minor_breaks = custom_minor_breaks) +
    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
      limits = c(0, NA)) +
    scale_colour_hue(name = "", h.start = 90,
      breaks = c("total", "total_guard", "total_exit", "reachable_guard",
        "reachable_exit", "exiting"),
      labels = c("Total (IPv4) OR", "Guard total (IPv4)", "Exit total (IPv4)",
        "Reachable guard IPv6 OR", "Reachable exit IPv6 OR", "IPv6 exiting")) +
    ggtitle("Advertised bandwidth by IP version") +
    labs(caption = copyright_notice) +
    theme(legend.position = "top") +
    guides(colour = guide_legend(nrow = 2, byrow = TRUE))
  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
}

write_advbw_ipv6 <- function(start_p = NULL, end_p = NULL, path_p) {
  prepare_advbw_ipv6(start_p, end_p) %>%
    rename(date = valid_after_date) %>%
    spread(category, advbw) %>%
    write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
}

