diff --git a/howto/gitlab.md b/howto/gitlab.md index cdcefd737a1ad8d51fe9a4c97143a7114807bcd9..5c1eeaf76ea27efe1eb5d33191cc16c7b8dccda0 100644 --- a/howto/gitlab.md +++ b/howto/gitlab.md @@ -620,6 +620,31 @@ To limit this to `job.log`, of course, you can do: find -name "job.log" -mtime +14 -print0 | du --files0-from=- -c -h | tee find-mtime+14-joblog-du.log +### Email routing + +Incoming email get routed through either eugeni or the submission +service, then end up on the Postfix server on `gitlab-02`, and from +there, to a dovecot mailbox. You can use `postfix-trace` to confirm +the message correctly ended up there. + +Normally, GitLab should be picking mails from the mailbox +(`/srv/mail/git@gitlab.torproject.org/Maildir/`) regularly, and +deleting them when done. If that is not happening, look at the +mailroom logs: + + tail -f /var/log/gitlab/mailroom/mail_room_json.log | jq -c + +A working run will look something like this: + +``` +{"severity":"INFO","time":"2022-08-29T20:15:57.734+00:00","context":{"email":"git@gitlab.torproject.org","name":"inbox"},"action":"Processing started"} +{"severity":"INFO","time":"2022-08-29T20:15:57.734+00:00","context":{"email":"git@gitlab.torproject.org","name":"inbox"},"uid":7788,"action":"asking arbiter to deliver","arbitrator":"MailRoom::Arbitration::Redis"}.734+00:00","context":{"email":"git@gitlab.torproject.org","name":"inbox"},"action":"Getting new messages","unread":{"count":1,"ids":[7788]},"to_be_delivered":{"count":1,"ids":[7788]}}ext":{"email":"git@gitlab.torproject.org","name":"inbox"},"uid":7788,"action":"sending to deliverer","deliverer":"MailRoom::Delivery::Sidekiq","byte_size":4162}","delivery_method":"Sidekiq","action":"message pushed"} +{"severity":"INFO","time":"2022-08-29T20:15:57.744+00:00","context":{"email":"git@gitlab.torproject.org","name":"inbox"},"action":"Processing started"} +{"severity":"INFO","time":"2022-08-29T20:15:57.744+00:00","context":{"email":"git@gitlab.torproject.org","name":"inbox"},"action":"Getting new messages","unread":{"count":0,"ids":[]},"to_be_delivered":{"count":0,"ids":[]}}0","context":{"email":"git@gitlab.torproject.org","name":"inbox"},"action":"Idling"} +``` + +Emails should be processed every minute or so. + ## Disaster recovery In case the entire GitLab machine is destroyed, a new server should be @@ -1163,8 +1188,27 @@ the production server. ## Logs and metrics -<!-- TODO: where are the logs? how long are they kept? any PII? --> -<!-- what about performance metrics? same questions --> +GitLab keeps an extensive (excessive?) amount of logs, in +`/var/log/gitlab`, which includes PII, including IP addresses. + +To see live logs, you can type the handy command: + + gitlab-ctl tail + +... but that is sort of like drinking from a fire hose. You can +inspect the logs of a specific component by passing it as an argument, +for example to inspect the mail importer: + + gitlab-ctl tail mailroom + +Each component is in his own directory, so the equivalent to the above +is: + + tail -f /var/log/gitlab/mailroom/{current,mail_room_json.log} + +Notice how both regular and JSON logs are kept. + +Logs seem to be kept for a month. ## Backups