awesome-go/stale_repositories_test.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
	"testing"
	"text/template"
	"time"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/oauth2"
)

const issueTemplateContent = `
{{range .}}
- [ ] {{.}}
{{end}}
`

var issueTemplate = template.Must(template.New("issue").Parse(issueTemplateContent))

// FIXME: replace to official github client
var reGithubRepo = regexp.MustCompile("https://github.com/[a-zA-Z0-9-._]+/[a-zA-Z0-9-._]+$")
var githubGETREPO = "https://api.github.com/repos%s"
var githubGETCOMMITS = "https://api.github.com/repos%s/commits"
var githubPOSTISSUES = "https://api.github.com/repos/avelino/awesome-go/issues"

// FIXME: use https
var awesomeGoGETISSUES = "http://api.github.com/repos/avelino/awesome-go/issues" //only returns open issues
// FIXME: time.Hour * ...
var numberOfYears time.Duration = 1
var timeNow = time.Now()
var issueTitle = fmt.Sprintf("Investigate repositories with more than 1 year without update - %s", timeNow.Format("2006-01-02"))

const deadLinkMessage = " this repository might no longer exist! (status code >= 400 returned)"
const movedPermanently = " status code 301 received"
const status302 = " status code 302 received"
const archived = " repository has been archived"

// LIMIT specifies the max number of repositories that are added in a single run of the script
var LIMIT = 10
var ctr = 0

type tokenSource struct {
	AccessToken string
}
type issue struct {
	Title string `json:"title"`
	Body  string `json:"body"`
}
type repo struct {
	Archived bool `json:"archived"`
}

func (t *tokenSource) Token() (*oauth2.Token, error) {
	token := &oauth2.Token{
		AccessToken: t.AccessToken,
	}
	return token, nil
}
func getRepositoriesFromBody(body string) []string {
	links := strings.Split(body, "- ")
	for idx, link := range links {
		str := strings.ReplaceAll(link, "\r", "")
		str = strings.ReplaceAll(str, "[ ]", "")
		str = strings.ReplaceAll(str, "[x]", "")
		str = strings.ReplaceAll(str, " ", "")
		str = strings.ReplaceAll(str, "\n", "")
		str = strings.ReplaceAll(str, deadLinkMessage, "")
		str = strings.ReplaceAll(str, movedPermanently, "")
		str = strings.ReplaceAll(str, status302, "")
		str = strings.ReplaceAll(str, archived, "")
		links[idx] = str
	}
	return links
}
func generateIssueBody(repositories []string) (string, error) {
	var writer bytes.Buffer

	err := issueTemplate.Execute(&writer, repositories)
	if err != nil {
		log.Print("Failed to generate template")
		return "", err
	}
	issueBody := writer.String()
	return issueBody, nil
}
func createIssue(staleRepos []string, client *http.Client) {
	if len(staleRepos) == 0 {
		log.Print("NO STALE REPOSITORIES")
		return
	}
	body, err := generateIssueBody(staleRepos)
	if err != nil {
		log.Print("Failed at CreateIssue")
		return
	}
	newIssue := &issue{
		Title: issueTitle,
		Body:  body,
	}
	buf := new(bytes.Buffer)
	json.NewEncoder(buf).Encode(newIssue)
	req, err := http.NewRequest("POST", githubPOSTISSUES, buf)
	if err != nil {
		log.Print("Failed at CreateIssue")
		return
	}
	client.Do(req)
}
func getAllFlaggedRepositories(client *http.Client, flaggedRepositories *map[string]bool) error {
	req, err := http.NewRequest("GET", awesomeGoGETISSUES, nil)
	if err != nil {
		log.Print("Failed to get all issues")
		return err
	}
	res, err := client.Do(req)
	if err != nil {
		log.Print("Failed to get all issues")
		return err
	}
	target := []issue{}
	defer res.Body.Close()
	json.NewDecoder(res.Body).Decode(&target)
	for _, i := range target {
		if i.Title == issueTitle {
			repos := getRepositoriesFromBody(i.Body)
			for _, repo := range repos {
				(*flaggedRepositories)[repo] = true
			}
		}
	}
	return nil
}
func containsOpenIssue(link string, openIssues map[string]bool) bool {
	_, ok := openIssues[link]
	return ok
}
func testRepoState(toRun bool, href string, client *http.Client, staleRepos *[]string) bool {
	if toRun {
		ownerRepo := strings.ReplaceAll(href, "https://github.com", "")
		apiCall := fmt.Sprintf(githubGETREPO, ownerRepo)
		req, err := http.NewRequest("GET", apiCall, nil)
		var repoResp repo
		isRepoAdded := false
		if err != nil {
			log.Printf("Failed at repository %s\n", href)
			return false
		}
		resp, err := client.Do(req)
		if err != nil {
			log.Printf("Failed at repository %s\n", href)
			return false
		}
		defer resp.Body.Close()
		json.NewDecoder(resp.Body).Decode(&repoResp)
		if resp.StatusCode == http.StatusMovedPermanently {
			*staleRepos = append(*staleRepos, href+movedPermanently)
			log.Printf("%s returned %d", href, resp.StatusCode)
			isRepoAdded = true
		}
		if resp.StatusCode == http.StatusFound && !isRepoAdded {
			*staleRepos = append(*staleRepos, href+status302)
			log.Printf("%s returned %d", href, resp.StatusCode)
			isRepoAdded = true
		}
		if resp.StatusCode >= http.StatusBadRequest && !isRepoAdded {
			*staleRepos = append(*staleRepos, href+deadLinkMessage)
			log.Printf("%s might not exist!", href)
			isRepoAdded = true
		}
		if repoResp.Archived && !isRepoAdded {
			*staleRepos = append(*staleRepos, href+archived)
			log.Printf("%s is archived!", href)
			isRepoAdded = true
		}
		return isRepoAdded
	}
	return false
}
func testCommitAge(toRun bool, href string, client *http.Client, staleRepos *[]string) bool {
	if toRun {
		var respObj []map[string]interface{}
		since := timeNow.Add(-1 * 365 * 24 * numberOfYears * time.Hour)
		sinceQuery := since.Format(time.RFC3339)
		ownerRepo := strings.ReplaceAll(href, "https://github.com", "")
		apiCall := fmt.Sprintf(githubGETCOMMITS, ownerRepo)
		req, err := http.NewRequest("GET", apiCall, nil)
		isRepoAdded := false
		if err != nil {
			log.Printf("Failed at repository %s\n", href)
			return false
		}
		q := req.URL.Query()
		q.Add("since", sinceQuery)
		req.URL.RawQuery = q.Encode()
		resp, err := client.Do(req)
		if err != nil {
			log.Printf("Failed at repository %s\n", href)
			return false
		}
		defer resp.Body.Close()
		json.NewDecoder(resp.Body).Decode(&respObj)
		isAged := len(respObj) == 0
		if isAged {
			log.Printf("%s has not had a commit in a while", href)
			*staleRepos = append(*staleRepos, href)
			isRepoAdded = true
		}
		return isRepoAdded
	}
	return false
}

func TestStaleRepository(t *testing.T) {
	doc := goqueryFromReadme(t)
	var staleRepos []string
	addressedRepositories := make(map[string]bool)
	oauth := os.Getenv("OAUTH_TOKEN")
	client := &http.Client{}
	if oauth == "" {
		log.Print("No oauth token found. Using unauthenticated client ...")
	} else {
		tokenSource := &tokenSource{
			AccessToken: oauth,
		}
		client = oauth2.NewClient(context.Background(), tokenSource)
	}
	err := getAllFlaggedRepositories(client, &addressedRepositories)

	if err != nil {
		log.Println("Failed to get existing issues. Exiting...")
		return
	}
	doc.Find("body li > a:first-child").EachWithBreak(func(_ int, s *goquery.Selection) bool {
		href, ok := s.Attr("href")
		if !ok {
			log.Println("expected to have href")
			return true
		}
		if ctr >= LIMIT && LIMIT != -1 {
			log.Print("Max number of issues created")
			return false
		}
		issueExists := containsOpenIssue(href, addressedRepositories)
		if issueExists {
			log.Printf("issue already exists for %s\n", href)
		} else {
			isGithubRepo := reGithubRepo.MatchString(href)
			if isGithubRepo {
				isRepoAdded := testRepoState(true, href, client, &staleRepos)
				isRepoAdded = testCommitAge(!isRepoAdded, href, client, &staleRepos)
				if isRepoAdded {
					ctr++
				}
			} else {
				log.Printf("%s non-github repo not currently handled", href)
			}
		}
		return true
	})
	createIssue(staleRepos, client)
}