Parse the new dirreq stats format. (e31189ae) · Commits · Legacy / gitolite / user / sjm217 / metrics

src/org/torproject/metrics/dirreq/ParseGeoipStats.java

+88 −30

Original line number	Diff line number	Diff line
		@@ -16,8 +16,14 @@ public final class ParseGeoipStats {
		SortedMap<String, Integer> v3Ips;
		SortedMap<String, Integer> v2Reqs;
		SortedMap<String, Integer> v3Reqs;
		String v2Resp;
		String v3Resp;
		int v2Share;
		int v3Share;
		String v2DirectDl;
		String v3DirectDl;
		String v2TunneledDl;
		String v3TunneledDl;
		}

		private static SortedSet<String> allCountries = new TreeSet<String>();
		@@ -40,8 +46,24 @@ public final class ParseGeoipStats {
		return result;
		}

		private static String toCommaSeparatedString(String line, int values) {
		StringBuilder out = new StringBuilder();
		String[] parts = line.split(" ")[1].split(",");
		for (String s : parts) {
		String[] p = s.split("=");
		out.append("," + p[1]);
		}
		for (int i = parts.length; i < values; i++) {
		out.append(",NA");
		}
		return out.toString();
		}

		private static String estimateRequestsAndClients(int localRequests,
		int localIpsInt, int shareAsInt) {
		if (shareAsInt == 0) {
		return "NA,NA,NA";
		}
		double share = ((double) shareAsInt) / 10000 * 5 / 4;
		double totalRequests = (double) localRequests / share;
		double totalClients = 10000.0D;
		@@ -109,45 +131,59 @@ public final class ParseGeoipStats {
		DataPoint currentDataPoint = null;
		boolean haveSeenActualNumbers = false;
		while ((line = br.readLine()) != null) {
		if (line.startsWith("written ")) {
		if (line.startsWith("dirreq-stats-end ")) {
		if (haveSeenActualNumbers) {
		currentDataPoints.put(currentDate, currentDataPoint);
		}
		currentDataPoint = new DataPoint();
		currentDate = line.split(" ")[1];
		allDates.add(currentDate);
		} else if (line.startsWith("started-at ")) {
		// ignored
		} else if (line.startsWith("ns-ips ")) {
		} else if (line.startsWith("dirreq-v3-ips ")) {
		currentDataPoint.v3Ips = parseCountryLine(line);
		if (line.split(" ").length > 1) {
		haveSeenActualNumbers = true;
		}
		} else if (line.startsWith("ns-v2-ips ")) {
		} else if (line.startsWith("dirreq-v2-ips ")) {
		currentDataPoint.v2Ips = parseCountryLine(line);
		if (line.split(" ").length > 1) {
		haveSeenActualNumbers = true;
		}
		} else if (line.startsWith("requests-start ")) {
		// ignored
		} else if (line.startsWith("n-ns-reqs ")) {
		} else if (line.startsWith("dirreq-v3-reqs ")) {
		currentDataPoint.v3Reqs = parseCountryLine(line);
		if (line.split(" ").length > 1) {
		haveSeenActualNumbers = true;
		}
		} else if (line.startsWith("n-v2-ns-reqs ")) {
		} else if (line.startsWith("dirreq-v2-reqs ")) {
		currentDataPoint.v2Reqs = parseCountryLine(line);
		if (line.split(" ").length > 1) {
		haveSeenActualNumbers = true;
		}
		} else if (line.startsWith("v2-ns-share ")) {
		} else if (line.startsWith("dirreq-v3-resp ")) {
		currentDataPoint.v3Resp =
		toCommaSeparatedString(line, 6);
		} else if (line.startsWith("dirreq-v2-resp ")) {
		currentDataPoint.v2Resp =
		toCommaSeparatedString(line, 5);
		} else if (line.startsWith("dirreq-v2-share ")) {
		currentDataPoint.v2Share = Integer.parseInt(
		line.split(" ")[1].replace('.', ';')
		.replace('%', ';').replaceAll(";", ""));
		} else if (line.startsWith("v3-ns-share ")) {
		} else if (line.startsWith("dirreq-v3-share ")) {
		currentDataPoint.v3Share = Integer.parseInt(
		line.split(" ")[1].replace('.', ';')
		.replace('%', ';').replaceAll(";", ""));
		} else if (line.startsWith("dirreq-v3-direct-dl ")) {
		currentDataPoint.v3DirectDl =
		toCommaSeparatedString(line, 16);
		} else if (line.startsWith("dirreq-v2-direct-dl ")) {
		currentDataPoint.v2DirectDl =
		toCommaSeparatedString(line, 16);
		} else if (line.startsWith("dirreq-v3-tunneled-dl ")) {
		currentDataPoint.v3TunneledDl =
		toCommaSeparatedString(line, 16);
		} else if (line.startsWith("dirreq-v2-tunneled-dl ")) {
		currentDataPoint.v2TunneledDl =
		toCommaSeparatedString(line, 16);
		}
		}
		if (haveSeenActualNumbers) {
		@@ -160,15 +196,11 @@ public final class ParseGeoipStats {
		+ "directories.%n", allCountries.size(), allDates.size(),
		allDataPoints.size());

		for (Map.Entry<String, SortedMap<String, DataPoint>> e
		: allDataPoints.entrySet()) {
		String directory = e.getKey();
		SortedMap<String, DataPoint> dataPoints = e.getValue();
		File outFile = new File(outputDirectory.getAbsolutePath()
		+ File.separatorChar + directory + ".csv");
		+ File.separatorChar + "dirreq.csv");
		BufferedWriter out = new BufferedWriter(new FileWriter(
		outFile, false));
		out.write("time,");
		out.write("directory,time,");
		for (String f : allCountries) {
		out.write(String.format("ip2%s,ip3%<s,ipt%<s,"
		+ "req2%<s,req3%<s,reqt%<s,", f));
		@@ -177,18 +209,38 @@ public final class ParseGeoipStats {
		+ "req2total,req3total,reqttotal,"
		+ "v2share,v3share,"
		+ "req2estimate,ip2estimate,reqperip2,"
		+ "req3estimate,ip3estimate,reqperip3\n");
		+ "req3estimate,ip3estimate,reqperip3,"
		+ "v2ok,v2unav,v2nfound,v2nmod,v2busy,"
		+ "v3ok,v3nsigs,v3unav,v3nfound,v3nmod,v3busy,"
		+ "v2dcompl,v2dtimeo,v2drunn,"
		+ "v2dmin,v2dd1,v2dd2,v2dq1,v2dd3,v2dd4,v2dmd,"
		+ "v2dd6,v2dd7,v2dq3,v2dd8,v2dd9,v2dmax,"
		+ "v2tcompl,v2ttimeo,v2trunn,"
		+ "v2tmin,v2td1,v2td2,v2tq1,v2td3,v2td4,v2tmd,"
		+ "v2td6,v2td7,v2tq3,v2td8,v2td9,v2tmax,"
		+ "v3dcompl,v3dtimeo,v3drunn,"
		+ "v3dmin,v3dd1,v3dd2,v3dq1,v3dd3,v3dd4,v3dmd,"
		+ "v3dd6,v3dd7,v3dq3,v3dd8,v3dd9,v3dmax,"
		+ "v3tcompl,v3ttimeo,v3trunn,"
		+ "v3tmin,v3td1,v3td2,v3tq1,v3td3,v3td4,v3tmd,"
		+ "v3td6,v3td7,v3tq3,v3td8,v3td9,v3tmax\n");
		for (Map.Entry<String, SortedMap<String, DataPoint>> e
		: allDataPoints.entrySet()) {
		String directory = e.getKey();
		SortedMap<String, DataPoint> dataPoints = e.getValue();


		for (String date : allDates) {
		if (!dataPoints.containsKey(date)) {
		out.write(date + ",");
		int nas = allCountries.size() * 6 + 7;
		out.write(directory + "," + date + ",");
		int nas = allCountries.size() * 6 + 13 + 6+5+16*4;
		for (int i = 0; i < nas; i++) {
		out.write("NA,");
		}
		out.write("NA\n");
		} else {
		DataPoint currentDataPoint = dataPoints.get(date);
		out.write(date + ",");
		out.write(directory + "," + date + ",");
		int ip2total = 0, ip3total = 0, ipttotal = 0,
		req2total = 0, req3total = 0, reqttotal = 0;
		for (String f : allCountries) {
		@@ -218,13 +270,19 @@ public final class ParseGeoipStats {
		out.write(String.format(",%s",
		estimateRequestsAndClients(req2total, ip2total,
		currentDataPoint.v2Share)));
		out.write(String.format(",%s%n",
		out.write(String.format(",%s",
		estimateRequestsAndClients(req3total, ip3total,
		currentDataPoint.v3Share)));
		out.write(currentDataPoint.v2Resp
		+ currentDataPoint.v3Resp
		+ currentDataPoint.v2DirectDl
		+ currentDataPoint.v2TunneledDl
		+ currentDataPoint.v3DirectDl
		+ currentDataPoint.v3TunneledDl + "\n");
		}
		}
		out.close();
		}
		out.close();

		System.out.println("Parsing finished after "
		+ ((System.currentTimeMillis() - started) / 1000)