Commit 7274522c authored by Arlo Breault's avatar Arlo Breault
Browse files

Move away from rsync

 * @kloesing shut down the server.

 * Closes #24 on github
parent ea8e165b
......@@ -3,10 +3,10 @@ SHELL := /bin/bash
start: exits i18n
@./check
rsync_server = metrics.torproject.org
consensuses_dir = metrics-recent/relay-descriptors/consensuses/
exit_lists_dir = metrics-recent/exit-lists/
descriptors_dir = metrics-recent/relay-descriptors/server-descriptors/
collector_url = https://collector.torproject.org/recent/
consensuses_dir = relay-descriptors/consensuses/
exit_lists_dir = exit-lists/
descriptors_dir = relay-descriptors/server-descriptors/
data/:
@mkdir -p data
......@@ -21,18 +21,27 @@ data/exit-lists/: data/
@mkdir -p data/exit-lists
data/consensus: data/consensuses/
@echo Getting latest consensus documents
@rsync -avz $(rsync_server)::$(consensuses_dir) --delete ./data/consensuses/
@echo "Getting latest consensus documents"
@pushd data/consensuses/; \
wget -r -nH -nd -nc --no-parent --reject "index.html*" \
$(collector_url)$(consensuses_dir); \
popd
@echo Consensuses written
data/exit-addresses: data/exit-lists/
@echo Getting latest exit lists
@rsync -avz $(rsync_server)::$(exit_lists_dir) --delete ./data/exit-lists/
@echo "Getting latest exit lists"
@pushd data/exit-lists/; \
wget -r -nH -nd -nc --no-parent --reject "index.html*" \
$(collector_url)$(exit_lists_dir); \
popd
@echo Exit lists written
exits: data/consensus data/exit-addresses data/cached-descriptors
@echo Generating exit-policies file
@python scripts/exitips.py
descriptors: data/descriptors/
@echo "Getting latest descriptors (This may take a while)"
@pushd data/descriptors/; \
wget -r -nH -nd -nc --no-parent --reject "index.html*" \
$(collector_url)$(descriptors_dir); \
popd
@echo Done
data/cached-descriptors: descriptors
......@@ -41,16 +50,30 @@ data/cached-descriptors: descriptors
find data/descriptors -type f -mmin -60 | xargs cat > data/cached-descriptors
@echo "Done"
descriptors_cutoff = $(shell date -v-1H -v-30M "+%Y/%m/%d %H:%M:%S")
descriptors: data/descriptors/
@echo "Getting latest descriptors (This may take a while)"
@find data/descriptors -type f -mmin +90 -delete
@rsync $(rsync_server)::$(descriptors_dir) | awk 'BEGIN { before="$(descriptors_cutoff)"; } before < ($$3 " " $$4) && ($$5 != ".") { print $$5; }' | rsync -avz --files-from=- $(rsync_server)::$(descriptors_dir) ./data/descriptors/
exits: data/consensus data/exit-addresses data/cached-descriptors
@echo Generating exit-policies file
@python scripts/exitips.py
@echo Done
locale/:
rm -rf locale
git clone -b torcheck_completed \
https://git.torproject.org/translation.git locale
pushd locale; \
for f in *; do \
if [ "$$f" != "templates" ]; then \
pushd "$$f"; \
mkdir LC_MESSAGES; \
msgfmt -o LC_MESSAGES/check.mo torcheck.po; \
popd; \
fi \
done
data/langs: data/
curl -k https://www.transifex.com/api/2/languages/ > data/langs
i18n: locale/ data/langs
build:
go fmt
go build
......@@ -68,22 +91,10 @@ bench: build
go test -benchtime 10s -bench "$(filter)" -benchmem
profile: build
go test -cpuprofile ../../cpu.prof -memprofile ../../mem.prof -benchtime 40s -bench "$(filter)"
i18n: locale/ data/langs
locale/:
rm -rf locale
git clone -b torcheck_completed https://git.torproject.org/translation.git locale
pushd locale; \
for f in *; do \
if [ "$$f" != "templates" ]; then \
pushd "$$f"; \
mkdir LC_MESSAGES; \
msgfmt -o LC_MESSAGES/check.mo torcheck.po; \
popd; \
fi \
done
go test \
-cpuprofile ../../cpu.prof \
-memprofile ../../mem.prof \
-benchtime 40s -bench "$(filter)"
install: build
mv check /usr/local/bin/check
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment