Merge remote-tracking branch 'gitlab/merge-requests/21' into maint-1.1

.gitchangelog.rc

+# -*- coding: utf-8; mode: python -*-
+##
+## Format
+##
+##   ACTION: [AUDIENCE:] COMMIT_MSG [!TAG ...]
+##
+## Description
+##
+##   ACTION is one of 'chg', 'fix', 'new'
+##
+##       Is WHAT the change is about.
+##
+##       'chg' is for refactor, small improvement, cosmetic changes...
+##       'fix' is for bug fixes
+##       'new' is for new features, big improvement
+##
+##   AUDIENCE is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'
+##
+##       Is WHO is concerned by the change.
+##
+##       'dev'  is for developpers (API changes, refactors...)
+##       'usr'  is for final users (UI changes)
+##       'pkg'  is for packagers   (packaging changes)
+##       'test' is for testers     (test only related changes)
+##       'doc'  is for doc guys    (doc only changes)
+##
+##   COMMIT_MSG is ... well ... the commit message itself.
+##
+##   TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic'
+##
+##       They are preceded with a '!' or a '@' (prefer the former, as the
+##       latter is wrongly interpreted in github.) Commonly used tags are:
+##
+##       'refactor' is obviously for refactoring code only
+##       'minor' is for a very meaningless change (a typo, adding a comment)
+##       'cosmetic' is for cosmetic driven change (re-indentation, 80-col...)
+##       'wip' is for partial functionality but complete subfunctionality.
+##
+## Example:
+##
+##   new: usr: support of bazaar implemented
+##   chg: re-indentend some lines !cosmetic
+##   new: dev: updated code to be compatible with last version of killer lib.
+##   fix: pkg: updated year of licence coverage.
+##   new: test: added a bunch of test around user usability of feature X.
+##   fix: typo in spelling my name in comment. !minor
+##
+##   Please note that multi-line commit message are supported, and only the
+##   first line will be considered as the "summary" of the commit message. So
+##   tags, and other rules only applies to the summary.  The body of the commit
+##   message will be displayed in the changelog without reformatting.
+
+
+##
+## ``ignore_regexps`` is a line of regexps
+##
+## Any commit having its full commit message matching any regexp listed here
+## will be ignored and won't be reported in the changelog.
+##
+ignore_regexps = [
+    r'@minor', r'!minor',
+    r'@cosmetic', r'!cosmetic',
+    r'@refactor', r'!refactor',
+    r'@wip', r'!wip',
+    r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[p|P]kg:',
+    r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[d|D]ev:',
+    r'^(.{3,3}\s*:)?\s*[fF]irst commit.?\s*$',
+    r'^$',  ## ignore commits with empty messages
+]
+
+
+## ``section_regexps`` is a list of 2-tuples associating a string label and a
+## list of regexp
+##
+## Commit messages will be classified in sections thanks to this. Section
+## titles are the label, and a commit is classified under this section if any
+## of the regexps associated is matching.
+##
+## Please note that ``section_regexps`` will only classify commits and won't
+## make any changes to the contents. So you'll probably want to go check
+## ``subject_process`` (or ``body_process``) to do some changes to the subject,
+## whenever you are tweaking this variable.
+##
+section_regexps = [
+    ('New', [
+        r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+     ]),
+    ('Changes', [
+        r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+     ]),
+    ('Fix', [
+        r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+     ]),
+
+    ('Other', None ## Match all lines
+     ),
+
+]
+
+
+## ``body_process`` is a callable
+##
+## This callable will be given the original body and result will
+## be used in the changelog.
+##
+## Available constructs are:
+##
+##   - any python callable that take one txt argument and return txt argument.
+##
+##   - ReSub(pattern, replacement): will apply regexp substitution.
+##
+##   - Indent(chars="  "): will indent the text with the prefix
+##     Please remember that template engines gets also to modify the text and
+##     will usually indent themselves the text if needed.
+##
+##   - Wrap(regexp=r"\n\n"): re-wrap text in separate paragraph to fill 80-Columns
+##
+##   - noop: do nothing
+##
+##   - ucfirst: ensure the first letter is uppercase.
+##     (usually used in the ``subject_process`` pipeline)
+##
+##   - final_dot: ensure text finishes with a dot
+##     (usually used in the ``subject_process`` pipeline)
+##
+##   - strip: remove any spaces before or after the content of the string
+##
+##   - SetIfEmpty(msg="No commit message."): will set the text to
+##     whatever given ``msg`` if the current text is empty.
+##
+## Additionally, you can `pipe` the provided filters, for instance:
+#body_process = Wrap(regexp=r'\n(?=\w+\s*:)') | Indent(chars="  ")
+#body_process = Wrap(regexp=r'\n(?=\w+\s*:)')
+#body_process = noop
+body_process = ReSub(r'((^|\n)[A-Z]\w+(-\w+)*: .*(\n\s+.*)*)+$', r'') | strip
+
+
+## ``subject_process`` is a callable
+##
+## This callable will be given the original subject and result will
+## be used in the changelog.
+##
+## Available constructs are those listed in ``body_process`` doc.
+subject_process = (strip |
+    ReSub(r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$', r'\4') |
+    SetIfEmpty("No commit message.") | ucfirst | final_dot)
+
+
+## ``tag_filter_regexp`` is a regexp
+##
+## Tags that will be used for the changelog must match this regexp.
+##
+## tag_filter_regexp = r'^[0-9]+\.[0-9]+(\.[0-9]+)?$'
+tag_filter_regexp = r'^v[0-9]+\.[0.9]+(\.[0-9]+)?$'
+
+## ``unreleased_version_label`` is a string or a callable that outputs a string
+##
+## This label will be used as the changelog Title of the last set of changes
+## between last valid tag and HEAD if any.
+unreleased_version_label = "Unreleased"
+
+## ``output_engine`` is a callable
+##
+## This will change the output format of the generated changelog file
+##
+## Available choices are:
+##
+##   - rest_py
+##
+##        Legacy pure python engine, outputs ReSTructured text.
+##        This is the default.
+##
+##   - mustache(<template_name>)
+##
+##        Template name could be any of the available templates in
+##        ``templates/mustache/*.tpl``.
+##        Requires python package ``pystache``.
+##        Examples:
+##           - mustache("markdown")
+##           - mustache("restructuredtext")
+##
+##   - makotemplate(<template_name>)
+##
+##        Template name could be any of the available templates in
+##        ``templates/mako/*.tpl``.
+##        Requires python package ``mako``.
+##        Examples:
+##           - makotemplate("restructuredtext")
+##
+output_engine = rest_py
+#output_engine = mustache("restructuredtext")
+#output_engine = mustache("markdown")
+#output_engine = makotemplate("restructuredtext")
+
+
+## ``include_merge`` is a boolean
+##
+## This option tells git-log whether to include merge commits in the log.
+## The default is to include them.
+# Including merge commit to be able to detect which group of commits belong
+# to the same feature/bug.
+include_merge = False
+
+
+## ``log_encoding`` is a string identifier
+##
+## This option tells gitchangelog what encoding is outputed by ``git log``.
+## The default is to be clever about it: it checks ``git config`` for
+## ``i18n.logOutputEncoding``, and if not found will default to git's own
+## default: ``utf-8``.
+#log_encoding = 'utf-8'
+
+
+## ``publish`` is a callable
+##
+## Sets what ``gitchangelog`` should do with the output generated by
+## the output engine. ``publish`` is a callable taking one argument
+## that is an interator on lines from the output engine.
+##
+## Some helper callable are provided:
+##
+## Available choices are:
+##
+##   - stdout
+##
+##        Outputs directly to standard output
+##        (This is the default)
+##
+##   - FileInsertAtFirstRegexMatch(file, pattern, idx=lamda m: m.start(), flags)
+##
+##        Creates a callable that will parse given file for the given
+##        regex pattern and will insert the output in the file.
+##        ``idx`` is a callable that receive the matching object and
+##        must return a integer index point where to insert the
+##        the output in the file. Default is to return the position of
+##        the start of the matched string.
+##
+##   - FileRegexSubst(file, pattern, replace, flags)
+##
+##        Apply a replace inplace in the given file. Your regex pattern must
+##        take care of everything and might be more complex. Check the README
+##        for a complete copy-pastable example.
+##
+publish = FileInsertAtFirstRegexMatch(
+    "CHANGELOG.rst",
+    r"(?P<rev>v[0-9]+\.[0-9]+(\.[0-9]+)?)\s+\([0-9]+-[0-9]{2}-[0-9]{2}\)\n--+\n",
+    idx=lambda m: m.start(1)
+)
+#publish = stdout
+
+
+## ``revs`` is a list of callable or a list of string
+##
+## callable will be called to resolve as strings and allow dynamical
+## computation of these. The result will be used as revisions for
+## gitchangelog (as if directly stated on the command line). This allows
+## to filter exaclty which commits will be read by gitchangelog.
+##
+## To get a full documentation on the format of these strings, please
+## refer to the ``git rev-list`` arguments. There are many examples.
+##
+## Using callables is especially useful, for instance, if you
+## are using gitchangelog to generate incrementally your changelog.
+##
+## Some helpers are provided, you can use them::
+##
+##   - FileFirstRegexMatch(file, pattern): will return a callable that will
+##     return the first string match for the given pattern in the given file.
+##     If you use named sub-patterns in your regex pattern, it'll output only
+##     the string matching the regex pattern named "rev".
+##
+##   - Caret(rev): will return the rev prefixed by a "^", which is a
+##     way to remove the given revision and all its ancestor.
+##
+## Please note that if you provide a rev-list on the command line, it'll
+## replace this value (which will then be ignored).
+##
+## If empty, then ``gitchangelog`` will act as it had to generate a full
+## changelog.
+##
+## The default is to use all commits to make the changelog.
+#revs = ["^1.0.3", ]
+revs = [
+    Caret(
+        FileFirstRegexMatch(
+            "CHANGELOG.rst",
+            r"(?P<rev>v[0-9]+\.[0-9]+(\.[0-9]+)?)\s+\([0-9]+-[0-9]{2}-[0-9]{2}\)\n--+\n")),
+    "HEAD"
+]
+#revs = []

.gitchangelogrst.tpl

+% if data["title"]:
+${data["title"]}
+\n
+${"=" * len(data["title"])}
+
+
+% endif
+% for version in data["versions"]:
+<%
+title = "%s (%s)" % (version["tag"], version["date"]) if version["tag"] else opts["unreleased_version_label"]
+
+nb_sections = len(version["sections"])
+%>${title}
+${"-" * len(title)}
+% for section in version["sections"]:
+% if not (section["label"] == "Other" and nb_sections == 1):
+
+${section["label"]}
+${"~" * len(section["label"])}
+% endif
+% for commit in section["commits"]:
+<%
+subject = "%s [%s]" % (commit["subject"], ", ".join(commit["authors"]))
+entry = indent('\n'.join(textwrap.wrap(subject)),
+                       first="- ").strip()
+%>${entry}
+
+% if commit["body"]:
+${indent(commit["body"])}
+% endif
+% endfor
+% endfor
+
+% endfor

docs/source/contributing.rst

@@ -135,6 +135,19 @@ Commits
 
 Each commit should reference the Tor Project Trac ticket (example: ``#12345``)
 and possibly the bugfix version.
+The commit message should contain ``Closes: #bugnumber``.
+
+From version 1.0.2 we started to prefix the summary with the subpackage or
+component, though we have not standarized the words to use, eg: ``scanner``,
+``generate``, ``v3bwfile``, ``relaylist``, ``doc``, ``test``, ``CI``.
+
+From version 1.0.3, we also started to prefix the summary with ``new``,
+``fix`` or ``chg``, so that `gitchangelog`_ automatically generates different
+sections in the CHANGELOG.
+
+From version 1.1.0 we started to use the words ``new``, ``chg`` and ``fix``,
+not in the sense `gitchangelog`_ use them, but to match sematic versioning
+changes major, minor and patch.
 
 Try to make each commit a logically separate changes.::
 
@@ -259,6 +272,10 @@ Each entry should reference the Tor Project Trac ticket (example: ``#12345``)
 and possibly the bugfix version.
 Until version 1.0.2 we have followed `keep a changelog`_ format.
 
+From version 1.1.x, run ``./scripts/maint/release.py`` to create new releases.
+It uses `gitchangelog`_ to automatically add new CHANGELOG entries from the
+commits' messages.
+
 .. _commit-msg:
 
 Example commit message
@@ -295,3 +312,4 @@ Example commit message
 .. _`keep a changelog`: https://keepachangelog.com/en/1.0.0/
 .. _`semantic versioning`: https://semver.org/
 .. _`vulture`: https://pypi.org/project/vulture/
+.. _`gitchangelog`: https://github.com/vaab/gitchangelog

scripts/maint/release.py

+#!/usr/bin/env python3
+# Copyright 2019 juga (juga at riseup dot net), CC0 license.
+"""Script to help release new versions.
+
+usage: release.py GPG_KEY_IDENTIFIER
+
+This is a fast and dirty script to help release new sbws versions.
+It does not intent to cover all possible cases and errors because this is out
+of the scope of sbws.
+There are other tools [*] that help manage release tags, branches and
+changelogs, but we should not add more requirements to sbws to just release a
+new version.
+
+This script will:
+0. Detect the current program version
+1. Ask which version to release
+2. Update the changelog automatically with ``gitchangelog``
+   and leave the option to user to manually edit it
+3. Commit the changelog
+4. Create a version with the tag and sign it
+5. Push the commit and tag to the repository
+6. Obtain the release tarball
+7. Sign the release tarball
+8. Modify the program to the next prerelease version and commit it
+9. Push the commit
+
+All is done in sequence and doesn't recover from any previous step.
+
+It assumes that:
+- the program version can be obtained with ``sbws.__version__``
+- ``gitchangelog`` and ``semantic_version`` python packages are installed
+- the official tarball releases are at gitlab.torproject.org
+- the key to sign the release is only one and is available in the system
+
+[*] See for instance https://semantic-release.gitbook.io,
+    https://github.com/GitTools/GitVersion
+
+"""
+import re
+import subprocess
+import sys
+try:
+    import semantic_version
+except ImportError:
+    print("Please, install semantic_version")
+    sys.exit(1)
+try:
+    import gitchangelog  # noqa
+except ImportError:
+    print("Please, install gitchangelog")
+    sys.exit(1)
+
+import sbws
+
+
+def obtain_release_version(version):
+    release_type_dict = {
+        1: version.next_patch(),
+        2: version.next_minor(),
+        3: version.next_major(),
+    }
+    print("Current prerelease version: ", version)
+    print("which version would you like to release?:")
+    print("1. to patch release: ", release_type_dict[1])
+    print("2. to minor release: ", release_type_dict[2])
+    print("3. to major release: ", release_type_dict[3])
+    release_type = int(input())
+    try:
+        return release_type_dict[release_type]
+    except KeyError:
+        print("Invalid release.")
+        sys.exit(1)
+
+
+def main(args):
+    print(__doc__)
+    try:
+        keyid = args[0]
+    except IndexError:
+        print("Please, pass a GPG key identifier as argument.")
+        sys.exit(1)
+    print("1. Which version to release")
+    print("---------------------------")
+    current_version = semantic_version.Version(sbws.__version__)
+    release_version = obtain_release_version(current_version)
+
+    print("\n2. Update changelog")
+    print("-------------------")
+    print("Creating tag v{} so that gitchangelog will create the new section "
+          "with the correct new tag...".format(release_version))
+    subprocess.call(['git', 'tag', 'v{}'.format(release_version),
+                     '-m', '\"Release version {}.\"'.format(release_version)])
+
+    print("\nUpdating the changelog automatically...")
+    subprocess.call('gitchangelog'.split(' '))
+
+    print("\nEdit the changelog manually to remove merge commits and "
+          "leave only the apropiate paragraph for every bug/feature.")
+    input("Press enter when done.\n")
+
+    print("\nRemoving the tag...")
+    subprocess.call(['git', 'tag', '-d', 'v{}'.format(release_version)])
+
+    print("\n3. Commit the changelog")
+    print("--------------------------")
+    print("\nCommiting CHANGELOG.rst...")
+    subprocess.call(['git', 'commit',
+                     '-am', '"Release version {}."'.format(release_version)])
+
+    print("\n4. Create tag and sign it")
+    print("--------------------------")
+    print("\nCreating the final tag and signing it...")
+    subprocess.call(['git', 'tag', '-s', 'v{}'.format(release_version),
+                     '-m', '"Release version {}."'.format(release_version)])
+
+    print("\n5. Push commit and tag")
+    print("------------------------")
+    print("\nPush now so that the Gitlab creates the tarball from the new "
+          " commit and tag, eg:")
+    print("git push myremote mybranch")
+    print("git push myremote --tags")
+    input("Press enter when you are done.")
+
+    print("\n6. Obtain the release tarball")
+    print("-------------------------------")
+    print("Obtaining Gitlab tarball...")
+    subprocess.call(
+        "wget https://gitlab.torproject.org/tpo/network-health/sbws/-/archive/v{}/sbws-v{}.tar.gz "
+        "-O v{}.tar.gz"
+        .format(release_version, release_version, release_version).split(' ')
+        )
+
+    print("\n7. Create the tarball signature")
+    print("-------------------------------")
+    print("Creating detached signature...")
+    subprocess.call("gpg --default-key {} "
+                    "--output v{}.tar.gz.asc "
+                    "--detach-sign v{}.tar.gz"
+                    .format(keyid, release_version, release_version)
+                    .split(' '))
+
+    print("\nUpload the signature manually to Gitlab.")
+    input("Press enter when done.")
+    print("\nRelease done!!!.")
+
+    print("\n8. Create next prerelease branch")
+    print("----------------------------------")
+    print("\nIf this release happens in a maintainance branch, merge the "
+          "the commit to master and push, eg:"
+          "git checkout master"
+          "git merge --no-ff mybranch"
+          "git push myremote master")
+    next_branch_version = "maint{}".format(release_version)
+    print("And create the next prerelease branch, eg:")
+          "git checkout -b {}".format(next_branch_version)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])