awesome-go/stale_repositories_test.go

310 lines
7.8 KiB
Go
Raw Normal View History

package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"regexp"
"strings"
"testing"
"text/template"
"time"
"github.com/PuerkitoBio/goquery"
"golang.org/x/oauth2"
)
2023-02-14 18:12:57 +00:00
const issueTemplateContent = `
{{range .}}
- [ ] {{.}}
{{end}}
`
2023-02-14 18:12:57 +00:00
var issueTemplate = template.Must(template.New("issue").Parse(issueTemplateContent))
2023-02-15 01:33:18 +00:00
// FIXME: replace to official github client
var reGithubRepo = regexp.MustCompile("https://github.com/[a-zA-Z0-9-._]+/[a-zA-Z0-9-._]+$")
var githubGETREPO = "https://api.github.com/repos%s"
var githubGETCOMMITS = "https://api.github.com/repos%s/commits"
var githubPOSTISSUES = "https://api.github.com/repos/avelino/awesome-go/issues"
2023-02-15 01:33:18 +00:00
// FIXME: use https
var awesomeGoGETISSUES = "http://api.github.com/repos/avelino/awesome-go/issues" //only returns open issues
2023-02-15 01:33:18 +00:00
// FIXME: time.Hour * ...
var numberOfYears time.Duration = 1
var timeNow = time.Now()
var issueTitle = fmt.Sprintf("Investigate repositories with more than 1 year without update - %s", timeNow.Format("2006-01-02"))
const deadLinkMessage = " this repository might no longer exist! (status code >= 400 returned)"
const movedPermanently = " status code 301 received"
const status302 = " status code 302 received"
const archived = " repository has been archived"
2023-02-14 18:12:57 +00:00
// LIMIT specifies the max number of repositories that are added in a single run of the script
var LIMIT = 10
var ctr = 0
type tokenSource struct {
AccessToken string
}
2023-02-15 01:39:11 +00:00
type issue struct {
Title string `json:"title"`
Body string `json:"body"`
}
2023-02-15 01:39:11 +00:00
type repo struct {
Archived bool `json:"archived"`
}
func (t *tokenSource) Token() (*oauth2.Token, error) {
2023-02-15 02:55:29 +00:00
return &oauth2.Token{
AccessToken: t.AccessToken,
2023-02-15 02:55:29 +00:00
}, nil
}
2023-02-15 01:39:11 +00:00
func getRepositoriesFromBody(body string) []string {
links := strings.Split(body, "- ")
2023-02-15 02:55:29 +00:00
for i, link := range links {
link = strings.ReplaceAll(link, "\r", "")
link = strings.ReplaceAll(link, "[ ]", "")
link = strings.ReplaceAll(link, "[x]", "")
link = strings.ReplaceAll(link, " ", "")
link = strings.ReplaceAll(link, "\n", "")
link = strings.ReplaceAll(link, deadLinkMessage, "")
link = strings.ReplaceAll(link, movedPermanently, "")
link = strings.ReplaceAll(link, status302, "")
link = strings.ReplaceAll(link, archived, "")
links[i] = link
}
2023-02-15 02:55:29 +00:00
return links
}
2023-02-15 01:39:11 +00:00
2023-02-15 01:38:24 +00:00
func generateIssueBody(t *testing.T, repositories []string) (string, error) {
t.Helper()
2023-02-14 18:12:57 +00:00
2023-02-15 01:38:24 +00:00
buf := bytes.NewBuffer(nil)
err := issueTemplate.Execute(buf, repositories)
requireNoErr(t, err, "Failed to generate template")
return buf.String(), nil
}
2023-02-15 01:38:24 +00:00
func createIssue(t *testing.T, staleRepos []string, client *http.Client) {
t.Helper()
if len(staleRepos) == 0 {
log.Print("NO STALE REPOSITORIES")
return
}
2023-02-15 01:38:24 +00:00
body, err := generateIssueBody(t, staleRepos)
requireNoErr(t, err, "failed to generate issue body")
newIssue := &issue{
Title: issueTitle,
Body: body,
}
2023-02-15 02:55:29 +00:00
buf := bytes.NewBuffer(nil)
2023-02-15 01:38:24 +00:00
requireNoErr(t, json.NewEncoder(buf).Encode(newIssue), "failed to encode json req")
req, err := http.NewRequest("POST", githubPOSTISSUES, buf)
2023-02-15 01:38:24 +00:00
requireNoErr(t, err, "failed to create request")
_, roundTripErr := client.Do(req)
requireNoErr(t, roundTripErr, "failed to send request")
}
2023-02-15 01:38:24 +00:00
// FIXME: remove pointer from map
func getAllFlaggedRepositories(t *testing.T, client *http.Client, flaggedRepositories *map[string]bool) error {
t.Helper()
// FIXME: replace to http.MethodGet
req, err := http.NewRequest("GET", awesomeGoGETISSUES, nil)
2023-02-15 01:38:24 +00:00
requireNoErr(t, err, "failed to create request")
res, err := client.Do(req)
2023-02-15 01:38:24 +00:00
requireNoErr(t, err, "failed to send request")
defer res.Body.Close()
2023-02-15 01:38:24 +00:00
2023-02-15 02:55:29 +00:00
var issues []issue
requireNoErr(t, json.NewDecoder(res.Body).Decode(&issues), "failed to unmarshal response")
2023-02-15 01:38:24 +00:00
2023-02-15 02:55:29 +00:00
for _, issue := range issues {
if issue.Title != issueTitle {
continue
}
repos := getRepositoriesFromBody(issue.Body)
for _, repo := range repos {
(*flaggedRepositories)[repo] = true
}
}
2023-02-15 02:55:29 +00:00
return nil
}
2023-02-15 01:39:11 +00:00
func containsOpenIssue(link string, openIssues map[string]bool) bool {
_, ok := openIssues[link]
return ok
}
2023-02-15 01:39:11 +00:00
func testRepoState(toRun bool, href string, client *http.Client, staleRepos *[]string) bool {
2023-02-15 02:55:29 +00:00
if !toRun {
return false
}
ownerRepo := strings.ReplaceAll(href, "https://github.com", "")
apiCall := fmt.Sprintf(githubGETREPO, ownerRepo)
req, err := http.NewRequest("GET", apiCall, nil)
if err != nil {
log.Printf("Failed at repository %s\n", href)
return false
}
resp, err := client.Do(req)
if err != nil {
log.Printf("Failed at repository %s\n", href)
return false
}
2023-02-15 02:55:29 +00:00
defer resp.Body.Close()
var repoResp repo
json.NewDecoder(resp.Body).Decode(&repoResp)
isRepoAdded := false
if resp.StatusCode == http.StatusMovedPermanently {
*staleRepos = append(*staleRepos, href+movedPermanently)
log.Printf("%s returned %d", href, resp.StatusCode)
isRepoAdded = true
}
if resp.StatusCode == http.StatusFound && !isRepoAdded {
*staleRepos = append(*staleRepos, href+status302)
log.Printf("%s returned %d", href, resp.StatusCode)
isRepoAdded = true
}
if resp.StatusCode >= http.StatusBadRequest && !isRepoAdded {
*staleRepos = append(*staleRepos, href+deadLinkMessage)
log.Printf("%s might not exist!", href)
isRepoAdded = true
}
if repoResp.Archived && !isRepoAdded {
*staleRepos = append(*staleRepos, href+archived)
log.Printf("%s is archived!", href)
isRepoAdded = true
}
return isRepoAdded
}
2023-02-15 01:39:11 +00:00
func testCommitAge(toRun bool, href string, client *http.Client, staleRepos *[]string) bool {
2023-02-15 02:55:29 +00:00
if !toRun {
return false
}
ownerRepo := strings.ReplaceAll(href, "https://github.com", "")
apiCall := fmt.Sprintf(githubGETCOMMITS, ownerRepo)
req, err := http.NewRequest("GET", apiCall, nil)
if err != nil {
log.Printf("Failed at repository %s\n", href)
return false
}
2023-02-15 02:55:29 +00:00
since := timeNow.Add(-1 * 365 * 24 * numberOfYears * time.Hour)
sinceQuery := since.Format(time.RFC3339)
q := req.URL.Query()
q.Add("since", sinceQuery)
req.URL.RawQuery = q.Encode()
resp, err := client.Do(req)
if err != nil {
log.Printf("Failed at repository %s\n", href)
return false
}
defer resp.Body.Close()
var respObj []map[string]interface{}
json.NewDecoder(resp.Body).Decode(&respObj)
isRepoAdded := false
isAged := len(respObj) == 0
if isAged {
log.Printf("%s has not had a commit in a while", href)
*staleRepos = append(*staleRepos, href)
isRepoAdded = true
}
return isRepoAdded
}
2023-02-15 00:14:43 +00:00
func TestStaleRepository(t *testing.T) {
doc := goqueryFromReadme(t)
2023-02-15 02:55:29 +00:00
oauth := os.Getenv("OAUTH_TOKEN")
2023-02-15 02:55:29 +00:00
client := &http.Client{
Transport: &http.Transport{},
}
if oauth == "" {
log.Print("No oauth token found. Using unauthenticated client ...")
} else {
tokenSource := &tokenSource{
AccessToken: oauth,
}
client = oauth2.NewClient(context.Background(), tokenSource)
}
2023-02-15 02:55:29 +00:00
2023-02-15 01:38:24 +00:00
addressedRepositories := make(map[string]bool)
2023-02-15 02:55:29 +00:00
// FIXME: return addressedRepositories, no need to pass
2023-02-15 01:38:24 +00:00
err := getAllFlaggedRepositories(t, client, &addressedRepositories)
requireNoErr(t, err, "failed to get existing issues")
2023-02-15 02:55:29 +00:00
var staleRepos []string
doc.
Find("body li > a:first-child").
EachWithBreak(func(_ int, s *goquery.Selection) bool {
href, ok := s.Attr("href")
if !ok {
log.Println("expected to have href")
return true
}
if ctr >= LIMIT && LIMIT != -1 {
log.Print("Max number of issues created")
return false
}
issueExists := containsOpenIssue(href, addressedRepositories)
if issueExists {
log.Printf("issue already exists for %s\n", href)
return true
}
isGithubRepo := reGithubRepo.MatchString(href)
2023-02-15 02:55:29 +00:00
if !isGithubRepo {
log.Printf("%s non-github repo not currently handled", href)
}
2023-02-15 02:55:29 +00:00
// FIXME: this is `or` expression. Probably we need `and`
isRepoAdded := testRepoState(true, href, client, &staleRepos)
isRepoAdded = testCommitAge(!isRepoAdded, href, client, &staleRepos)
if isRepoAdded {
ctr++
}
return true
})
2023-02-15 01:38:24 +00:00
createIssue(t, staleRepos, client)
}