Commit ce9094e

Eric Bower  ·  2023-08-15 10:13:04 -0400 EDT
parent 9365859 
refactor: perf improvements
1 files changed,  +234, -187
M main.go
M main.go
+234, -187
......@@ -13,6 +13,7 @@ import (
1313 "path/filepath"
1414 "sort"
1515 "strings"
16+ "sync"
1617 "unicode/utf8"
1718
1819 "github.com/alecthomas/chroma"
......@@ -61,6 +62,8 @@ type Config struct {
6162 // computed
6263 // cache for skipping commits, trees, etc.
6364 Cache map[string]bool
65+ // mutex for Cache
66+ Mutex sync.RWMutex
6467 // pretty name for the repo
6568 RepoName string
6669 // logger
......@@ -107,6 +110,7 @@ type CommitData struct {
107110 WhenStr string
108111 AuthorStr string
109112 ShortID string
113+ ParentID string
110114 Refs []*RefInfo
111115 *git.Commit
112116 }
......@@ -301,31 +305,6 @@ func readmeFile(repo *Config) string {
301305 return strings.ToLower(repo.Readme)
302306 }
303307
304-func walkTree(tree *git.Tree, revData *RevData, curpath string, aggregate []*TreeItem) []*TreeItem {
305- entries, err := tree.Entries()
306- bail(err)
307-
308- for _, entry := range entries {
309- fname := filepath.Join(curpath, entry.Name())
310- typ := entry.Type()
311- if typ == git.ObjectTree {
312- re, _ := tree.Subtree(entry.Name())
313- aggregate = walkTree(re, revData, fname, aggregate)
314- }
315-
316- if entry.Type() == git.ObjectBlob {
317- aggregate = append(aggregate, &TreeItem{
318- Size: toPretty(entry.Size()),
319- Path: fname,
320- Entry: entry,
321- URL: template.URL(getFileURL(revData, fname)),
322- })
323- }
324- }
325-
326- return aggregate
327-}
328-
329308 func (c *Config) writeHtml(writeData *WriteData) {
330309 ts, err := template.ParseFS(
331310 efs,
......@@ -357,6 +336,7 @@ func (c *Config) copyStatic(dst string, data []byte) {
357336 }
358337
359338 func (c *Config) writeRootSummary(data *PageData, readme template.HTML) {
339+ c.Logger.Infof("writing root html (%s)", c.RepoPath)
360340 c.writeHtml(&WriteData{
361341 Filename: "index.html",
362342 Template: "html/summary.page.tmpl",
......@@ -368,6 +348,7 @@ func (c *Config) writeRootSummary(data *PageData, readme template.HTML) {
368348 }
369349
370350 func (c *Config) writeTree(data *PageData, tree []*TreeItem) {
351+ c.Logger.Infof("writing tree (%s)", data.RevData.Name())
371352 c.writeHtml(&WriteData{
372353 Filename: "index.html",
373354 Subdir: getTreeBaseDir(data.RevData),
......@@ -380,6 +361,7 @@ func (c *Config) writeTree(data *PageData, tree []*TreeItem) {
380361 }
381362
382363 func (c *Config) writeLog(data *PageData, logs []*CommitData) {
364+ c.Logger.Infof("writing log file (%s)", data.RevData.Name())
383365 c.writeHtml(&WriteData{
384366 Filename: "index.html",
385367 Subdir: getLogBaseDir(data.RevData),
......@@ -392,6 +374,7 @@ func (c *Config) writeLog(data *PageData, logs []*CommitData) {
392374 }
393375
394376 func (c *Config) writeRefs(data *PageData, refs []*RefInfo) {
377+ c.Logger.Infof("writing refs (%s)", c.RepoPath)
395378 c.writeHtml(&WriteData{
396379 Filename: "refs.html",
397380 Template: "html/refs.page.tmpl",
......@@ -402,125 +385,114 @@ func (c *Config) writeRefs(data *PageData, refs []*RefInfo) {
402385 })
403386 }
404387
405-func (c *Config) writeHTMLTreeFiles(pageData *PageData, tree []*TreeItem) string {
388+func (c *Config) writeHTMLTreeFile(pageData *PageData, treeItem *TreeItem) string {
406389 readme := ""
407- for _, file := range tree {
408- b, err := file.Entry.Blob().Bytes()
409- bail(err)
410- str := string(b)
411-
412- file.IsTextFile = isTextFile(str)
390+ b, err := treeItem.Entry.Blob().Bytes()
391+ bail(err)
392+ str := string(b)
413393
414- contents := "binary file, cannot display"
415- if file.IsTextFile {
416- file.NumLines = len(strings.Split(str, "\n"))
417- contents, err = parseText(file.Entry.Name(), string(b), c.Theme)
418- bail(err)
419- }
394+ treeItem.IsTextFile = isTextFile(str)
420395
421- d := filepath.Dir(file.Path)
396+ contents := "binary file, cannot display"
397+ if treeItem.IsTextFile {
398+ treeItem.NumLines = len(strings.Split(str, "\n"))
399+ contents, err = parseText(treeItem.Entry.Name(), string(b), c.Theme)
400+ bail(err)
401+ }
422402
423- nameLower := strings.ToLower(file.Entry.Name())
424- summary := readmeFile(pageData.Repo)
425- if nameLower == summary {
426- readme = contents
427- }
403+ d := filepath.Dir(treeItem.Path)
428404
429- c.writeHtml(&WriteData{
430- Filename: fmt.Sprintf("%s.html", file.Entry.Name()),
431- Template: "html/file.page.tmpl",
432- Data: &FilePageData{
433- PageData: pageData,
434- Contents: template.HTML(contents),
435- Path: file.Path,
436- },
437- Subdir: getFileURL(pageData.RevData, d),
438- })
405+ nameLower := strings.ToLower(treeItem.Entry.Name())
406+ summary := readmeFile(pageData.Repo)
407+ if nameLower == summary {
408+ readme = contents
439409 }
410+
411+ c.writeHtml(&WriteData{
412+ Filename: fmt.Sprintf("%s.html", treeItem.Entry.Name()),
413+ Template: "html/file.page.tmpl",
414+ Data: &FilePageData{
415+ PageData: pageData,
416+ Contents: template.HTML(contents),
417+ Path: treeItem.Path,
418+ },
419+ Subdir: getFileURL(pageData.RevData, d),
420+ })
440421 return readme
441422 }
442423
443-func (c *Config) writeLogDiffs(repo *git.Repository, pageData *PageData, logs []*CommitData) {
444- for _, commit := range logs {
445- commitID := commit.ID.String()
424+func (c *Config) writeLogDiff(repo *git.Repository, pageData *PageData, commit *CommitData) {
425+ commitID := commit.ID.String()
446426
447- if c.Cache[commitID] {
448- c.Logger.Infof("(%s) commit file already generated, skipping", getShortID(commitID))
449- continue
450- } else {
451- c.Cache[commitID] = true
452- }
427+ c.Mutex.RLock()
428+ hasCommit := c.Cache[commitID]
429+ c.Mutex.RUnlock()
453430
454- ancestors, err := commit.Ancestors()
455- bail(err)
431+ if hasCommit {
432+ c.Logger.Infof("(%s) commit file already generated, skipping", getShortID(commitID))
433+ return
434+ } else {
435+ c.Mutex.Lock()
436+ c.Cache[commitID] = true
437+ c.Mutex.Unlock()
438+ }
456439
457- // if no ancestors exist then we are at initial commit
458- parent := commit
459- if len(ancestors) > 0 {
460- pt := ancestors[0]
461- parent = &CommitData{
462- Commit: pt,
463- URL: getCommitURL(pt.ID.String()),
464- }
465- }
466- parentID := parent.ID.String()
467-
468- diff, err := repo.Diff(
469- commitID,
470- 0,
471- 0,
472- 0,
473- git.DiffOptions{Base: parentID},
474- )
475-
476- rnd := &DiffRender{
477- NumFiles: diff.NumFiles(),
478- TotalAdditions: diff.TotalAdditions(),
479- TotalDeletions: diff.TotalDeletions(),
480- }
481- fls := []*DiffRenderFile{}
482- for _, file := range diff.Files {
483- fl := &DiffRenderFile{
484- FileType: diffFileType(file.Type),
485- OldMode: file.OldMode(),
486- OldName: file.OldName(),
487- Mode: file.Mode(),
488- Name: file.Name,
489- NumAdditions: file.NumAdditions(),
490- NumDeletions: file.NumDeletions(),
491- }
492- content := ""
493- for _, section := range file.Sections {
494- for _, line := range section.Lines {
495- content += fmt.Sprintf("%s\n", line.Content)
496- }
497- }
498- // set filename to something our `ParseText` recognizes (e.g. `.diff`)
499- finContent, err := parseText("commit.diff", content, c.Theme)
500- bail(err)
440+ diff, err := repo.Diff(
441+ commitID,
442+ 0,
443+ 0,
444+ 0,
445+ git.DiffOptions{},
446+ )
447+ bail(err)
501448
502- fl.Content = template.HTML(finContent)
503- fls = append(fls, fl)
449+ rnd := &DiffRender{
450+ NumFiles: diff.NumFiles(),
451+ TotalAdditions: diff.TotalAdditions(),
452+ TotalDeletions: diff.TotalDeletions(),
453+ }
454+ fls := []*DiffRenderFile{}
455+ for _, file := range diff.Files {
456+ fl := &DiffRenderFile{
457+ FileType: diffFileType(file.Type),
458+ OldMode: file.OldMode(),
459+ OldName: file.OldName(),
460+ Mode: file.Mode(),
461+ Name: file.Name,
462+ NumAdditions: file.NumAdditions(),
463+ NumDeletions: file.NumDeletions(),
504464 }
505- rnd.Files = fls
506-
507- commitData := &CommitPageData{
508- PageData: pageData,
509- Commit: commit,
510- CommitID: getShortID(commitID),
511- Diff: rnd,
512- Parent: getShortID(parentID),
513- CommitURL: getCommitURL(commitID),
514- ParentURL: getCommitURL(parentID),
465+ content := ""
466+ for _, section := range file.Sections {
467+ for _, line := range section.Lines {
468+ content += fmt.Sprintf("%s\n", line.Content)
469+ }
515470 }
471+ // set filename to something our `ParseText` recognizes (e.g. `.diff`)
472+ finContent, err := parseText("commit.diff", content, c.Theme)
473+ bail(err)
516474
517- c.writeHtml(&WriteData{
518- Filename: fmt.Sprintf("%s.html", commitID),
519- Template: "html/commit.page.tmpl",
520- Subdir: "commits",
521- Data: commitData,
522- })
475+ fl.Content = template.HTML(finContent)
476+ fls = append(fls, fl)
477+ }
478+ rnd.Files = fls
479+
480+ commitData := &CommitPageData{
481+ PageData: pageData,
482+ Commit: commit,
483+ CommitID: getShortID(commitID),
484+ Diff: rnd,
485+ Parent: getShortID(commit.ParentID),
486+ CommitURL: getCommitURL(commitID),
487+ ParentURL: getCommitURL(commit.ParentID),
523488 }
489+
490+ c.writeHtml(&WriteData{
491+ Filename: fmt.Sprintf("%s.html", commitID),
492+ Template: "html/commit.page.tmpl",
493+ Subdir: "commits",
494+ Data: commitData,
495+ })
524496 }
525497
526498 func getSummaryURL() template.URL {
......@@ -585,6 +557,7 @@ func getShortID(id string) string {
585557 }
586558
587559 func (c *Config) writeRepo() *BranchOutput {
560+ c.Logger.Infof("Writing repo (%s)", c.RepoPath)
588561 repo, err := git.Open(c.RepoPath)
589562 bail(err)
590563
......@@ -664,6 +637,7 @@ func (c *Config) writeRepo() *BranchOutput {
664637 })
665638
666639 for _, revData := range revs {
640+ c.Logger.Infof("Writing revision (%s)", revData.Name())
667641 data := &PageData{
668642 Repo: c,
669643 RevData: revData,
......@@ -698,6 +672,37 @@ func (c *Config) writeRepo() *BranchOutput {
698672 return mainOutput
699673 }
700674
675+type TreeWalker struct {
676+ revData *RevData
677+ treeItem chan *TreeItem
678+}
679+
680+func (tw *TreeWalker) walk(tree *git.Tree, curpath string) {
681+ entries, err := tree.Entries()
682+ bail(err)
683+
684+ for _, entry := range entries {
685+ fname := filepath.Join(curpath, entry.Name())
686+ typ := entry.Type()
687+
688+ if typ == git.ObjectTree {
689+ re, _ := tree.Subtree(entry.Name())
690+ tw.walk(re, fname)
691+ } else if typ == git.ObjectBlob {
692+ tw.treeItem <- &TreeItem{
693+ Size: toPretty(entry.Size()),
694+ Path: fname,
695+ Entry: entry,
696+ URL: template.URL(getFileURL(tw.revData, fname)),
697+ }
698+ }
699+ }
700+
701+ if curpath == "" {
702+ close(tw.treeItem)
703+ }
704+}
705+
701706 func (c *Config) writeRevision(repo *git.Repository, pageData *PageData, refs []*RefInfo) *BranchOutput {
702707 c.Logger.Infof(
703708 "compiling (%s) revision (%s)",
......@@ -706,89 +711,131 @@ func (c *Config) writeRevision(repo *git.Repository, pageData *PageData, refs []
706711 )
707712
708713 output := &BranchOutput{}
709- pageSize := pageData.Repo.MaxCommits
710- if pageSize == 0 {
711- pageSize = 5000
712- }
713714
714- commits, err := repo.CommitsByPage(pageData.RevData.ID(), 0, pageSize)
715- bail(err)
715+ var wg sync.WaitGroup
716716
717- logs := []*CommitData{}
718- for i, commit := range commits {
719- if i == 0 {
720- output.LastCommit = commit
717+ wg.Add(1)
718+ go func() {
719+ defer wg.Done()
720+
721+ pageSize := pageData.Repo.MaxCommits
722+ if pageSize == 0 {
723+ pageSize = 5000
721724 }
725+ fmt.Println("grabbing commits")
726+ commits, err := repo.CommitsByPage(pageData.RevData.ID(), 0, pageSize)
727+ fmt.Println("got commits")
728+ bail(err)
722729
723- tags := []*RefInfo{}
724- for _, ref := range refs {
725- if commit.ID.String() == ref.ID {
726- tags = append(tags, ref)
730+ logs := []*CommitData{}
731+ for i, commit := range commits {
732+ if i == 0 {
733+ output.LastCommit = commit
734+ }
735+
736+ tags := []*RefInfo{}
737+ for _, ref := range refs {
738+ if commit.ID.String() == ref.ID {
739+ tags = append(tags, ref)
740+ }
727741 }
742+
743+ parentSha, _ := commit.ParentID(0)
744+ parentID := ""
745+ if parentSha == nil {
746+ parentID = commit.ID.String()
747+ } else {
748+ parentID = parentSha.String()
749+ }
750+ logs = append(logs, &CommitData{
751+ ParentID: parentID,
752+ URL: getCommitURL(commit.ID.String()),
753+ ShortID: getShortID(commit.ID.String()),
754+ SummaryStr: commit.Summary(),
755+ AuthorStr: commit.Author.Name,
756+ WhenStr: commit.Author.When.Format("02 Jan 06"),
757+ Commit: commit,
758+ Refs: tags,
759+ })
728760 }
729761
730- logs = append(logs, &CommitData{
731- URL: getCommitURL(commit.ID.String()),
732- ShortID: getShortID(commit.ID.String()),
733- SummaryStr: commit.Summary(),
734- AuthorStr: commit.Author.Name,
735- WhenStr: commit.Author.When.Format("02 Jan 06"),
736- Commit: commit,
737- Refs: tags,
738- })
739- }
762+ c.writeLog(pageData, logs)
763+
764+ for _, cm := range logs {
765+ wg.Add(1)
766+ go func(commit *CommitData) {
767+ defer wg.Done()
768+ c.writeLogDiff(repo, pageData, commit)
769+ }(cm)
770+ }
771+ }()
740772
741773 tree, err := repo.LsTree(pageData.RevData.ID())
742774 bail(err)
743775
744- entries := []*TreeItem{}
745- treeEntries := walkTree(tree, pageData.RevData, "", entries)
746- for _, entry := range treeEntries {
747- entry.Path = strings.TrimPrefix(entry.Path, "/")
776+ treeEntries := []*TreeItem{}
777+ readme := ""
778+ entries := make(chan *TreeItem)
779+ tw := &TreeWalker{
780+ revData: pageData.RevData,
781+ treeItem: entries,
782+ }
783+ wg.Add(1)
784+ go func() {
785+ defer wg.Done()
786+ tw.walk(tree, "")
787+ }()
748788
749- var lastCommits []*git.Commit
750- // `git rev-list` is pretty expensive here, so we have a flag to disable
751- if pageData.Repo.HideTreeLastCommit {
752- c.Logger.Info("skipping the process of finding the last commit for each file")
753- } else {
754- lastCommits, err = repo.RevList([]string{pageData.RevData.ID()}, git.RevListOptions{
755- Path: entry.Path,
756- CommandOptions: git.CommandOptions{Args: []string{"-1"}},
757- })
758- bail(err)
789+ for e := range entries {
790+ wg.Add(1)
791+ go func(entry *TreeItem) {
792+ defer wg.Done()
793+ entry.Path = strings.TrimPrefix(entry.Path, "/")
794+
795+ var lastCommits []*git.Commit
796+ // `git rev-list` is pretty expensive here, so we have a flag to disable
797+ if pageData.Repo.HideTreeLastCommit {
798+ // c.Logger.Info("skipping the process of finding the last commit for each file")
799+ } else {
800+ lastCommits, err = repo.RevList([]string{pageData.RevData.ID()}, git.RevListOptions{
801+ Path: entry.Path,
802+ CommandOptions: git.CommandOptions{Args: []string{"-1"}},
803+ })
804+ bail(err)
805+
806+ var lc *git.Commit
807+ if len(lastCommits) > 0 {
808+ lc = lastCommits[0]
809+ }
810+ entry.CommitURL = getCommitURL(lc.ID.String())
811+ entry.Summary = lc.Summary()
812+ entry.When = lc.Author.When.Format("02 Jan 06")
813+ }
814+
815+ fpath := getFileURL(
816+ pageData.RevData,
817+ fmt.Sprintf("%s.html", entry.Path),
818+ )
819+ entry.URL = template.URL(fpath)
759820
760- var lc *git.Commit
761- if len(lastCommits) > 0 {
762- lc = lastCommits[0]
821+ readmeStr := c.writeHTMLTreeFile(pageData, entry)
822+ if readmeStr != "" {
823+ readme = readmeStr
763824 }
764- entry.CommitURL = getCommitURL(lc.ID.String())
765- entry.Summary = lc.Summary()
766- entry.When = lc.Author.When.Format("02 Jan 06")
767- }
768- fpath := getFileURL(
769- pageData.RevData,
770- fmt.Sprintf("%s.html", entry.Path),
771- )
772- entry.URL = template.URL(fpath)
825+ treeEntries = append(treeEntries, entry)
826+ }(e)
773827 }
774828
829+ wg.Wait()
830+
775831 c.Logger.Infof(
776832 "compilation complete (%s) branch (%s)",
777833 c.RepoName,
778834 pageData.RevData.Name(),
779835 )
780836
781- go func() {
782- c.writeLog(pageData, logs)
783- }()
784- go func() {
785- c.writeLogDiffs(repo, pageData, logs)
786- }()
787- go func() {
788- c.writeTree(pageData, treeEntries)
789- }()
837+ c.writeTree(pageData, treeEntries)
790838
791- readme := c.writeHTMLTreeFiles(pageData, treeEntries)
792839 output.Readme = readme
793840 return output
794841 }