releaser: Fix tag detection for changelog when doing a main release

Also improve the changelog slightly.

Fixes #3482

5 files changed, 102 insertions, 32 deletions

diff --git a/releaser/git.go b/releaser/git.go
index d8b5bef31..654ea78f0 100644
--- a/releaser/git.go
+++ b/releaser/git.go
@@ -25,6 +25,7 @@ import (
 var issueRe = regexp.MustCompile(`(?i)[Updates?|Closes?|Fix.*|See] #(\d+)`)
 
 const (
+	notesChanges    = "notesChanges"
 	templateChanges = "templateChanges"
 	coreChanges     = "coreChanges"
 	outChanges      = "outChanges"
@@ -36,6 +37,7 @@ type changeLog struct {
 	Version      string
 	Enhancements map[string]gitInfos
 	Fixes        map[string]gitInfos
+	Notes        gitInfos
 	All          gitInfos
 
 	// Overall stats
@@ -44,22 +46,25 @@ type changeLog struct {
 	ThemeCount       int
 }
 
-func newChangeLog(infos gitInfos) changeLog {
-	return changeLog{
+func newChangeLog(infos gitInfos) *changeLog {
+	return &changeLog{
 		Enhancements: make(map[string]gitInfos),
 		Fixes:        make(map[string]gitInfos),
 		All:          infos,
 	}
 }
 
-func (l changeLog) addGitInfo(isFix bool, info gitInfo, category string) {
+func (l *changeLog) addGitInfo(isFix bool, info gitInfo, category string) {
 	var (
 		infos   gitInfos
 		found   bool
 		segment map[string]gitInfos
 	)
 
-	if isFix {
+	if category == notesChanges {
+		l.Notes = append(l.Notes, info)
+		return
+	} else if isFix {
 		segment = l.Fixes
 	} else {
 		segment = l.Enhancements
@@ -74,7 +79,7 @@ func (l changeLog) addGitInfo(isFix bool, info gitInfo, category string) {
 	segment[category] = infos
 }
 
-func gitInfosToChangeLog(infos gitInfos) changeLog {
+func gitInfosToChangeLog(infos gitInfos) *changeLog {
 	log := newChangeLog(infos)
 	for _, info := range infos {
 		los := strings.ToLower(info.Subject)
@@ -82,7 +87,9 @@ func gitInfosToChangeLog(infos gitInfos) changeLog {
 		var category = otherChanges
 
 		// TODO(bep) improve
-		if regexp.MustCompile("(?i)tpl:|tplimpl:|layout").MatchString(los) {
+		if regexp.MustCompile("(?i)deprecate").MatchString(los) {
+			category = notesChanges
+		} else if regexp.MustCompile("(?i)tpl|tplimpl:|layout").MatchString(los) {
 			category = templateChanges
 		} else if regexp.MustCompile("(?i)docs?:|documentation:").MatchString(los) {
 			category = docsChanges
@@ -150,8 +157,8 @@ func git(args ...string) (string, error) {
 	return string(out), nil
 }
 
-func getGitInfos(remote bool) (gitInfos, error) {
-	return getGitInfosBefore("HEAD", remote)
+func getGitInfos(tag string, remote bool) (gitInfos, error) {
+	return getGitInfosBefore("HEAD", tag, remote)
 }
 
 type countribCount struct {
@@ -207,11 +214,11 @@ func (g gitInfos) ContribCountPerAuthor() contribCounts {
 	return c
 }
 
-func getGitInfosBefore(ref string, remote bool) (gitInfos, error) {
+func getGitInfosBefore(ref, tag string, remote bool) (gitInfos, error) {
 
 	var g gitInfos
 
-	log, err := gitLogBefore(ref)
+	log, err := gitLogBefore(ref, tag)
 	if err != nil {
 		return g, err
 	}
@@ -242,10 +249,16 @@ func getGitInfosBefore(ref string, remote bool) (gitInfos, error) {
 // Ignore autogenerated commits etc. in change log. This is a regexp.
 const ignoredCommits = "release:|vendor:|snapcraft:"
 
-func gitLogBefore(ref string) (string, error) {
-	prevTag, err := gitShort("describe", "--tags", "--abbrev=0", "--always", ref+"^")
-	if err != nil {
-		return "", err
+func gitLogBefore(ref, tag string) (string, error) {
+	var prevTag string
+	var err error
+	if tag != "" {
+		prevTag = tag
+	} else {
+		prevTag, err = gitVersionTagBefore(ref)
+		if err != nil {
+			return "", err
+		}
 	}
 	log, err := git("log", "-E", fmt.Sprintf("--grep=%s", ignoredCommits), "--invert-grep", "--pretty=format:%x1e%h%x1f%aE%x1f%s%x1f%b", "--abbrev-commit", prevTag+".."+ref)
 	if err != nil {
@@ -255,11 +268,29 @@ func gitLogBefore(ref string) (string, error) {
 	return log, err
 }
 
+func gitVersionTagBefore(ref string) (string, error) {
+	return gitShort("describe", "--tags", "--abbrev=0", "--always", "--match", "v[0-9]*", ref+"^")
+}
+
 func gitLog() (string, error) {
-	return gitLogBefore("HEAD")
+	return gitLogBefore("HEAD", "")
 }
 
 func gitShort(args ...string) (output string, err error) {
 	output, err = git(args...)
 	return strings.Replace(strings.Split(output, "\n")[0], "'", "", -1), err
 }
+
+func tagExists(tag string) (bool, error) {
+	out, err := git("tag", "-l", tag)
+
+	if err != nil {
+		return false, err
+	}
+
+	if strings.Contains(out, tag) {
+		return true, nil
+	}
+
+	return false, nil
+}
diff --git a/releaser/git_test.go b/releaser/git_test.go
index dc1db5dc7..1c102520e 100644
--- a/releaser/git_test.go
+++ b/releaser/git_test.go
@@ -14,19 +14,15 @@
 package releaser
 
 import (
+	"os"
 	"testing"
 
-	"runtime"
-
 	"github.com/stretchr/testify/require"
 )
 
 func TestGitInfos(t *testing.T) {
-	if runtime.GOOS == "linux" {
-		// Travis has an ancient git with no --invert-grep: https://github.com/travis-ci/travis-ci/issues/6328
-		t.Skip("Skip git test on Linux to make Travis happy.")
-	}
-	infos, err := getGitInfos(false)
+	skipIfCI(t)
+	infos, err := getGitInfos("v0.20", false)
 
 	require.NoError(t, err)
 	require.True(t, len(infos) > 0)
@@ -51,3 +47,30 @@ See #456
 	require.Equal(t, 543, issues[2])
 
 }
+
+func TestGitVersionTagBefore(t *testing.T) {
+	skipIfCI(t)
+	v1, err := gitVersionTagBefore("v0.18")
+	require.NoError(t, err)
+	require.Equal(t, "v0.17", v1)
+}
+
+func TestTagExists(t *testing.T) {
+	skipIfCI(t)
+	b1, err := tagExists("v0.18")
+	require.NoError(t, err)
+	require.True(t, b1)
+
+	b2, err := tagExists("adfagdsfg")
+	require.NoError(t, err)
+	require.False(t, b2)
+
+}
+
+func skipIfCI(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		// Travis has an ancient git with no --invert-grep: https://github.com/travis-ci/travis-ci/issues/6328
+		// Also Travis clones very shallowly, making some of the tests above shaky.
+		t.Skip("Skip git test on Linux to make Travis happy.")
+	}
+}
diff --git a/releaser/releasenotes_writer.go b/releaser/releasenotes_writer.go
index 3d48d9ae8..2f24a6c05 100644
--- a/releaser/releasenotes_writer.go
+++ b/releaser/releasenotes_writer.go
@@ -62,7 +62,10 @@ Hugo now has:
 {{ with .ThemeCount }}
 * 156+ [themes](http://themes.gohugo.io/)
 {{- end }}
-
+{{ with .Notes }}
+## Notes
+{{ template "change-section" . }}
+{{- end -}}
 ## Enhancements
 {{ template "change-headers"  .Enhancements -}}
 ## Fixes
@@ -80,7 +83,7 @@ Hugo now has:
 {{- end -}}
 {{- with $outChanges -}}
 ### Output
-{{- template "change-section"  . }}
+{{ template "change-section"  . }}
 {{- end -}}
 {{- with $coreChanges -}}
 ### Core
@@ -88,7 +91,7 @@ Hugo now has:
 {{- end -}}
 {{- with $docsChanges -}}
 ### Docs
-{{- template "change-section"  . }}
+{{ template "change-section"  . }}
 {{- end -}}
 {{- with $otherChanges -}}
 ### Other
diff --git a/releaser/releasenotes_writer_test.go b/releaser/releasenotes_writer_test.go
index 95e30e2a2..1b759b1d9 100644
--- a/releaser/releasenotes_writer_test.go
+++ b/releaser/releasenotes_writer_test.go
@@ -18,6 +18,7 @@ package releaser
 
 import (
 	"bytes"
+	"fmt"
 	"os"
 	"testing"
 
@@ -33,9 +34,11 @@ func _TestReleaseNotesWriter(t *testing.T) {
 	var b bytes.Buffer
 
 	// TODO(bep) consider to query GitHub directly for the gitlog with author info, probably faster.
-	infos, err := getGitInfosBefore("v0.20", false)
+	infos, err := getGitInfosBefore("HEAD", "v0.20", false)
 	require.NoError(t, err)
 
-	require.NoError(t, writeReleaseNotes("0.20", infos, &b))
+	require.NoError(t, writeReleaseNotes("0.21", infos, &b))
+
+	fmt.Println(b.String())
 
 }
diff --git a/releaser/releaser.go b/releaser/releaser.go
index ddf64a759..e7e7f479f 100644
--- a/releaser/releaser.go
+++ b/releaser/releaser.go
@@ -24,7 +24,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"strings"
 
 	"github.com/spf13/hugo/helpers"
 )
@@ -89,20 +88,31 @@ func (r *ReleaseHandler) Run() error {
 	tag := "v" + version
 
 	// Exit early if tag already exists
-	out, err := git("tag", "-l", tag)
-
+	exists, err := tagExists(tag)
 	if err != nil {
 		return err
 	}
 
-	if strings.Contains(out, tag) {
+	if exists {
 		return fmt.Errorf("Tag %q already exists", tag)
 	}
 
+	var changeLogFromTag string
+
+	if newVersion.PatchLevel == 0 {
+		// There may have been patch releases inbetween, so set the tag explicitly.
+		changeLogFromTag = "v" + newVersion.Prev().String()
+		exists, _ := tagExists(changeLogFromTag)
+		if !exists {
+			// fall back to one that exists.
+			changeLogFromTag = ""
+		}
+	}
+
 	var gitCommits gitInfos
 
 	if r.shouldPrepare() || r.shouldRelease() {
-		gitCommits, err = getGitInfos(true)
+		gitCommits, err = getGitInfos(changeLogFromTag, true)
 		if err != nil {
 			return err
 		}