commits-to-changelog script.

This script converts git commit messages to ChangeLog entries.  That
will hopefully allow us to forego adding ChangeLog entries in our
commits, which always leads to merge conflicts.

1 files changed, 254 insertions, 0 deletions

diff --git a/scripts/commits-to-changelog.py b/scripts/commits-to-changelog.py
new file mode 100755
index 00000000000..ef8ed099cb1
--- /dev/null
+++ b/scripts/commits-to-changelog.py
@@ -0,0 +1,254 @@
+#!/usr/bin/python
+
+from optparse import OptionParser
+import subprocess
+import re
+import os.path
+import fnmatch
+import os
+
+# subtract 8 for the leading tabstop
+fill_column = 74 - 8
+
+path_to_root = None
+
+all_changelogs = {}
+
+def git (command, *args):
+    return subprocess.Popen (["git", command] + list (args), stdout = subprocess.PIPE).communicate () [0]
+
+def changelog_path (changelog):
+    global path_to_root
+    if not path_to_root:
+        path_to_root = git ("rev-parse", "--show-cdup").strip ()
+    (pathname, filename) = changelog
+    return path_to_root + "./" + pathname + "/" + filename
+
+def changelog_for_file (filename):
+    while filename != "":
+        dirname = os.path.dirname (filename)
+        if dirname in all_changelogs:
+            return (dirname, all_changelogs [dirname])
+        filename = dirname
+    assert False
+
+def changelogs_for_file_pattern (pattern, changed_files):
+    changelogs = set ()
+    for filename in changed_files:
+        suffix = filename
+        while suffix != "":
+            # FIXME: fnmatch doesn't support the {x,y} pattern
+            if fnmatch.fnmatch (suffix, pattern):
+                changelogs.add (changelog_for_file (filename))
+            (_, _, suffix) = suffix.partition ("/")
+    return changelogs
+
+def format_paragraph (paragraph):
+    lines = []
+    words = paragraph.split ()
+    current = words [0]
+    for word in words [1:]:
+        if len (current) + 1 + len (word) <= fill_column:
+            current += " " + word
+        else:
+            lines.append ("\t" + current)
+            current = word
+    lines.append ("\t" + current)
+    return lines
+
+def format_changelog_paragraph (files, paragraph):
+    files_string = ""
+    for (filename, entity) in files:
+        if len (files_string) > 0:
+            files_string += ", "
+        files_string += filename
+        if entity:
+            files_string += " (" + entity + ")"
+    return format_paragraph ("* " + files_string + ": " + paragraph)
+
+def append_paragraph (lines, paragraph):
+    if len (lines):
+        lines.append ("")
+    lines += paragraph
+
+def format_changelog_entries (commit, changed_files, prefix, file_entries):
+    changelogs = set ()
+    for f in changed_files:
+        changelogs.add (changelog_for_file (f))
+    marked_changelogs = set ()
+
+    author_line = git ("log", "-n1", "--date=short", "--format=%ad  %an  <%ae>", commit).strip ()
+
+    paragraphs = {}
+    for changelog in changelogs:
+        paragraphs [changelog] = [author_line]
+
+    for (files, comments) in file_entries:
+        changelog_entries = {}
+        for (filename, entity) in files:
+            entry_changelogs = changelogs_for_file_pattern (filename, changed_files)
+            if len (entry_changelogs) == 0:
+                print "Warning: could not match file %s in commit %s" % (filename, commit)
+            for changelog in entry_changelogs:
+                if changelog not in changelog_entries:
+                    changelog_entries [changelog] = []
+                changelog_entries [changelog].append ((filename, entity))
+                marked_changelogs.add (changelog)
+
+        for (changelog, files) in changelog_entries.items ():
+            append_paragraph (paragraphs [changelog], format_changelog_paragraph (files, comments [0]))
+            for paragraph in comments [1:]:
+                append_paragraph (paragraphs [changelog], format_paragraph (paragraph))
+
+    unmarked_changelogs = changelogs - marked_changelogs
+    for changelog in unmarked_changelogs:
+        if len (prefix) == 0:
+            print "Warning: empty entry in %s for commit %s" % (changelog_path (changelog), commit)
+            append_paragraph (paragraphs [changelog], format_paragraph ("FIXME: empty entry!"))
+        for paragraph in prefix:
+            append_paragraph (paragraphs [changelog], format_paragraph (paragraph))
+
+    return paragraphs
+
+def debug_print_commit (commit, raw_message, prefix, file_entries, changed_files, changelog_entries):
+    print "===================== Commit"
+    print commit
+    print "--------------------- RAW"
+    print raw_message
+    print "--------------------- Prefix"
+    for line in prefix:
+        print line
+    print "--------------------- File entries"
+    for (files, comments) in file_entries:
+        files_str = ""
+        for (filename, entity) in files:
+            if len (files_str):
+                files_str = files_str + ", "
+            files_str = files_str + filename
+            if entity:
+                files_str = files_str + " (" + entity + ")"
+        print files_str
+        for line in comments:
+            print "  " + line
+    print "--------------------- Files touched"
+    for f in changed_files:
+        print f
+    print "--------------------- ChangeLog entries"
+    for ((dirname, filename), lines) in changelog_entries.items ():
+        print "%s/%s:" % (dirname, filename)
+        for line in lines:
+            print line
+
+def process_commit (commit):
+    changed_files = map (lambda l: l.split () [2], git ("diff-tree", "--numstat", commit).splitlines () [1:])
+    if len (filter (lambda f: re.search ("(^|/)Change[Ll]og$", f), changed_files)):
+        return
+    raw_message = git ("log", "-n1", "--format=%B", commit)
+    # filter SVN migration message
+    message = re.sub ("(^|\n)svn path=[^\n]+revision=\d+(?=$|\n)", "", raw_message)
+    # filter ChangeLog headers
+    message = re.sub ("(^|\n)\d+-\d+-\d+[ \t]+((\w|[.-])+[ \t]+)+<[^\n>]+>(?=$|\n)", "", message)
+    # filter leading whitespace
+    message = re.sub ("^\s+", "", message)
+    # filter trailing whitespace
+    message = re.sub ("\s+$", "", message)
+    # paragraphize - first remove whitespace at beginnings and ends of lines
+    message = re.sub ("[ \t]*\n[ \t]*", "\n", message)
+    # paragraphize - now replace three or more consecutive newlines with two
+    message = re.sub ("\n\n\n+", "\n\n", message)
+    # paragraphize - replace single newlines with a space
+    message = re.sub ("(?<!\n)\n(?!\n)", " ", message)
+    # paragraphize - finally, replace double newlines with single ones
+    message = re.sub ("\n\n", "\n", message)
+
+    # A list of paragraphs (strings without newlines) that occur
+    # before the first file comments
+    prefix = []
+
+    # A list of tuples of the form ([(filename, entity), ...], [paragraph, ...]).
+    #
+    # Each describes a file comment, containing multiple paragraphs.
+    # Those paragraphs belong to a list of files, each with an
+    # optional entity (usually a function name).
+    file_entries = []
+
+    current_files = None
+    current_files_comments = None
+
+    for line in message.splitlines ():
+        if re.match ("\*\s[^:]+:", line):
+            if current_files:
+                file_entries.append ((current_files, current_files_comments))
+
+            (files, _, comments) = line.partition (":")
+
+            current_files_comments = [comments.strip ()]
+
+            current_files = []
+            for f in re.split ("\s*,\s*", files [1:].strip ()):
+                m = re.search ("\(([^()]+)\)$", f)
+                if m:
+                    filename = f [:m.start (0)].strip ()
+                    entity = m.group (1).strip ()
+                else:
+                    filename = f
+                    entity = None
+                current_files.append ((filename, entity))
+        else:
+            if current_files:
+                current_files_comments.append (line)
+            else:
+                prefix.append (line)
+    if current_files:
+        file_entries.append ((current_files, current_files_comments))
+
+    changelog_entries = format_changelog_entries (commit, changed_files, prefix, file_entries)
+
+    #debug_print_commit (commit, raw_message, prefix, file_entries, changed_files, changelog_entries)
+
+    return changelog_entries
+
+def start_changelog (changelog):
+    full_path = changelog_path (changelog)
+    old_name = full_path + ".old"
+    os.rename (full_path, old_name)
+    return open (full_path, "w")
+
+def finish_changelog (changelog, file):
+    old_file = open (changelog_path (changelog) + ".old")
+    file.write (old_file.read ())
+    old_file.close ()
+    file.close ()
+
+def append_lines (file, lines):
+    for line in lines:
+        file.write (line + "\n")
+    file.write ("\n")
+
+def main ():
+    usage = "usage: %prog [options] <start-commit>"
+    parser = OptionParser (usage)
+    (options, args) = parser.parse_args ()
+    if len (args) != 1:
+        parser.error ("incorrect number of arguments")
+    start_commit = args [0]
+
+    for filename in git ("ls-tree", "-r", "--name-only", "HEAD").splitlines ():
+        if re.search ("(^|/)Change[Ll]og$", filename):
+            (path, name) = os.path.split (filename)
+            all_changelogs [path] = name
+
+    commits = git ("rev-list", "--no-merges", "HEAD", "^%s" % start_commit).splitlines ()
+
+    touched_changelogs = {}
+    for commit in commits:
+        entries = process_commit (commit)
+        for (changelog, lines) in entries.items ():
+            if changelog not in touched_changelogs:
+                touched_changelogs [changelog] = start_changelog (changelog)
+            append_lines (touched_changelogs [changelog], lines)
+    for (changelog, file) in touched_changelogs.items ():
+        finish_changelog (changelog, file)
+
+if __name__ == "__main__":
+    main ()