feat: Fetch tags from GitHub, use only latest from SDP release line

Makefile

@@ -1,5 +1,5 @@
 # Set the shell to bash always
-SHELL := /bin/bash
+SHELL := /usr/bin/env bash
 
 # Note: CGO_ENABLED is required for the SQLite3 module.
 export CGO_ENABLED=1

README.md

@@ -27,23 +27,18 @@ It stores these in an SQLite database.
 ## Generating docs
 
 Have a look at https://github.com/stackabletech/crddocs for sample usage.
-To generate docs you need a yaml configuration file specifying which repos and tags to document.
-It should look like this:
+To generate docs you need a yaml configuration file listing the repos to document:
 
     repos:
-      airflow-operator:
-        - "24.7.0"
-        - "nightly"
-      druid-operator:
-        - "24.7.0"
-        - "nightly"
-      hbase-operator:
-        - "24.7.0"
-        - "nightly"
-
-    platformVersions:
-      - "24.7.0"
-      - "nightly"
+      - airflow-operator
+      - druid-operator
+      - hbase-operator
+
+Tags are auto-discovered: all calver `YY.M.P` tags on each repo at
+`github.com/stackabletech/<repo>` are listed, only the latest patch per
+`YY.M` release line is kept, and `nightly` (tracking `main`) is always added.
+The platform-version list shown on the landing page is the union across all
+repos.
 
 You also need a HTML file template and a directory of static files.
 

docs-generator/doc/doc.go

@@ -157,6 +157,10 @@ func main() {
 		log.Fatalf("Error loading config: %s: %v", configFile, err)
 		panic(err)
 	}
+	if err := conf.ResolveTags(); err != nil {
+		log.Fatalf("Error resolving tags: %v", err)
+		panic(err)
+	}
 
 	// generate landing page(s)
 	home(db, outDir, "", conf.PlatformVersions)
@@ -165,7 +169,7 @@ func main() {
 	}
 
 	// generate doc pages for all repos and CRDs
-	for repo, tags := range conf.Repos {
+	for repo, tags := range conf.Tags {
 		org(db, outDir, repo, "")
 		for _, tag := range tags {
 			org(db, outDir, repo, tag)

docs-generator/gitter/gitter.go

@@ -72,9 +72,13 @@ func main() {
 		log.Fatalf("Error loading config: %s: %v", configFile, err)
 		panic(err)
 	}
+	if err := conf.ResolveTags(); err != nil {
+		log.Fatalf("Error resolving tags: %v", err)
+		panic(err)
+	}
 
 	// index repos
-	for repo, tags := range conf.Repos {
+	for repo, tags := range conf.Tags {
 		log.Printf("Indexing repo %s ...\n", repo)
 		for _, tag := range tags {
 			log.Printf("... at tag: %s ...\n", tag)

docs-generator/pkg/config/config.go

@@ -1,15 +1,28 @@
 package config
 
 import (
+	"fmt"
 	"log"
 	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
 
+	"github.com/go-git/go-git/v5"
+	gitconfig "github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/storage/memory"
 	"gopkg.in/yaml.v2"
 )
 
+const NightlyTag = "nightly"
+
 type Config struct {
-	Repos            map[string][]string `yaml:"repos"`
-	PlatformVersions []string            `yaml:"platformVersions"`
+	Repos []string `yaml:"repos"`
+
+	// Tags and PlatformVersions are populated by ResolveTags.
+	Tags             map[string][]string `yaml:"-"`
+	PlatformVersions []string            `yaml:"-"`
 }
 
 func (c *Config) NewConfigFromFile(filePath string) error {
@@ -19,11 +32,116 @@ func (c *Config) NewConfigFromFile(filePath string) error {
 		return err
 	}
 
-	err = yaml.Unmarshal(yamlFile, c)
-	if err != nil {
+	if err := yaml.Unmarshal(yamlFile, c); err != nil {
 		log.Fatalf("Error unmarshalling YAML: %v", err)
 		return err
 	}
 
 	return nil
 }
+
+// ResolveTags lists remote tags for each configured repo, keeps only the
+// latest patch per YY.M release line, appends `nightly`, and fills
+// c.Tags and c.PlatformVersions. PlatformVersions is the union across all
+// repos, newest calver first, with nightly last (matching the previous
+// hand-maintained order).
+func (c *Config) ResolveTags() error {
+	c.Tags = map[string][]string{}
+	platformSet := map[string]struct{}{}
+
+	for _, repo := range c.Repos {
+		url := "https://github.com/stackabletech/" + repo
+		log.Printf("Listing tags for %s ...", repo)
+		tags, err := listLatestPatches(url)
+		if err != nil {
+			return fmt.Errorf("listing tags for %s: %w", repo, err)
+		}
+		c.Tags[repo] = append(tags, NightlyTag)
+
+		for _, t := range tags {
+			platformSet[t] = struct{}{}
+		}
+	}
+
+	platform := make([]string, 0, len(platformSet))
+	for v := range platformSet {
+		platform = append(platform, v)
+	}
+	sortCalverDesc(platform)
+	c.PlatformVersions = append(platform, NightlyTag)
+
+	return nil
+}
+
+type calver struct{ y, m, p int }
+
+var calverRe = regexp.MustCompile(`^(\d{2})\.(\d{1,2})\.(\d+)$`)
+
+func parseCalver(s string) (calver, bool) {
+	m := calverRe.FindStringSubmatch(s)
+	if m == nil {
+		return calver{}, false
+	}
+	y, _ := strconv.Atoi(m[1])
+	mo, _ := strconv.Atoi(m[2])
+	p, _ := strconv.Atoi(m[3])
+	return calver{y, mo, p}, true
+}
+
+func calverLess(a, b calver) bool {
+	if a.y != b.y {
+		return a.y < b.y
+	}
+	if a.m != b.m {
+		return a.m < b.m
+	}
+	return a.p < b.p
+}
+
+func sortCalverDesc(versions []string) {
+	sort.Slice(versions, func(i, j int) bool {
+		a, _ := parseCalver(versions[i])
+		b, _ := parseCalver(versions[j])
+		return calverLess(b, a)
+	})
+}
+
+// listLatestPatches returns the latest-patch calver tag per YY.M release
+// line for the given remote, newest first.
+func listLatestPatches(url string) ([]string, error) {
+	rem := git.NewRemote(memory.NewStorage(), &gitconfig.RemoteConfig{
+		Name: "origin",
+		URLs: []string{url},
+	})
+	refs, err := rem.List(&git.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	type line struct{ y, m int }
+	latest := map[line]calver{}
+	names := map[line]string{}
+
+	for _, ref := range refs {
+		if !ref.Name().IsTag() {
+			continue
+		}
+		name := strings.TrimPrefix(ref.Name().String(), "refs/tags/")
+		cv, ok := parseCalver(name)
+		if !ok {
+			continue
+		}
+		key := line{cv.y, cv.m}
+		if cur, exists := latest[key]; !exists || calverLess(cur, cv) {
+			latest[key] = cv
+			names[key] = name
+		}
+	}
+
+	out := make([]string, 0, len(names))
+	for _, n := range names {
+		out = append(out, n)
+	}
+	sortCalverDesc(out)
+	return out, nil
+}