playground
/
gogs
mirror of https://github.com/gogits/gogs.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

cron: add repository archive cleanup (#4061)

pull/4150/merge
Unknwon 8 years ago
parent
c69900325d
commit
0a2f87f941
No known key found for this signature in database
GPG Key ID: 25B575AE3213B2B3
9 changed files with 115 additions and 28 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 7
      conf/app.ini
  2. 2
      gogs.go
  3. 97
      models/repo.go
  4. 4
      modules/bindata/bindata.go
  5. 17
      modules/cron/cron.go
  6. 6
      modules/setting/setting.go
  7. 2
      templates/.VERSION
  8. 2
      vendor/github.com/gogits/git-module/repo_commit.go
  9. 6
      vendor/vendor.json

7
conf/app.ini
Unescape View File

@ -341,6 +341,13 @@ ARGS =
RUN_AT_START = true
SCHEDULE = @every 24h
; Cleanup repository archives
[cron.repo_archive_cleanup]
RUN_AT_START = false
SCHEDULE = @every 24h
; Time duration to check if archive should be cleaned
OLDER_THAN = 24h
[git]
; Disables highlight of added and removed changes
DISABLE_DIFF_HIGHLIGHT = false

2
gogs.go
Unescape View File

@ -16,7 +16,7 @@ import (
"github.com/gogits/gogs/modules/setting"
)
const APP_VER = "0.9.158.0218"
const APP_VER = "0.9.159.0218"
func init() {
setting.AppVer = APP_VER

97
models/repo.go
Unescape View File

@ -1597,8 +1597,70 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (repos []*Repository, _ int
return repos, count, sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).Find(&repos)
}
func DeleteOldRepositoryArchives() {
if taskStatusTable.IsRunning(_CLEAN_OLD_ARCHIVES) {
return
}
taskStatusTable.Start(_CLEAN_OLD_ARCHIVES)
defer taskStatusTable.Stop(_CLEAN_OLD_ARCHIVES)
log.Trace("Doing: DeleteOldRepositoryArchives")
formats := []string{"zip", "targz"}
oldestTime := time.Now().Add(-setting.Cron.RepoArchiveCleanup.OlderThan)
if err := x.Where("id > 0").Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
basePath := filepath.Join(repo.RepoPath(), "archives")
for _, format := range formats {
dirPath := filepath.Join(basePath, format)
if !com.IsDir(dirPath) {
continue
}
dir, err := os.Open(dirPath)
if err != nil {
log.Error(3, "Fail to open directory '%s': %v", dirPath, err)
continue
}
fis, err := dir.Readdir(0)
dir.Close()
if err != nil {
log.Error(3, "Fail to read directory '%s': %v", dirPath, err)
continue
}
for _, fi := range fis {
if fi.IsDir() || fi.ModTime().After(oldestTime) {
continue
}
archivePath := filepath.Join(dirPath, fi.Name())
if err = os.Remove(archivePath); err != nil {
desc := fmt.Sprintf("Fail to health delete archive '%s': %v", archivePath, err)
log.Warn(desc)
if err = CreateRepositoryNotice(desc); err != nil {
log.Error(3, "CreateRepositoryNotice: %v", err)
}
}
}
}
return nil
}); err != nil {
log.Error(2, "DeleteOldRepositoryArchives: %v", err)
}
}
// DeleteRepositoryArchives deletes all repositories' archives.
func DeleteRepositoryArchives() error {
if taskStatusTable.IsRunning(_CLEAN_OLD_ARCHIVES) {
return nil
}
taskStatusTable.Start(_CLEAN_OLD_ARCHIVES)
defer taskStatusTable.Stop(_CLEAN_OLD_ARCHIVES)
return x.Where("id > 0").Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
@ -1688,9 +1750,10 @@ func SyncRepositoryHooks() error {
var taskStatusTable = sync.NewStatusTable()
const (
_MIRROR_UPDATE = "mirror_update"
_GIT_FSCK = "git_fsck"
_CHECK_REPOs = "check_repos"
_MIRROR_UPDATE = "mirror_update"
_GIT_FSCK = "git_fsck"
_CHECK_REPO_STATS = "check_repos_stats"
_CLEAN_OLD_ARCHIVES = "clean_old_archives"
)
// GitFsck calls 'git fsck' to check repository health.
@ -1708,15 +1771,15 @@ func GitFsck() {
repo := bean.(*Repository)
repoPath := repo.RepoPath()
if err := git.Fsck(repoPath, setting.Cron.RepoHealthCheck.Timeout, setting.Cron.RepoHealthCheck.Args...); err != nil {
desc := fmt.Sprintf("Fail to health check repository (%s): %v", repoPath, err)
desc := fmt.Sprintf("Fail to health check repository '%s': %v", repoPath, err)
log.Warn(desc)
if err = CreateRepositoryNotice(desc); err != nil {
log.Error(4, "CreateRepositoryNotice: %v", err)
log.Error(3, "CreateRepositoryNotice: %v", err)
}
}
return nil
}); err != nil {
log.Error(4, "GitFsck: %v", err)
log.Error(2, "GitFsck: %v", err)
}
}
@ -1747,7 +1810,7 @@ type repoChecker struct {
func repoStatsCheck(checker *repoChecker) {
results, err := x.Query(checker.querySQL)
if err != nil {
log.Error(4, "Select %s: %v", checker.desc, err)
log.Error(2, "Select %s: %v", checker.desc, err)
return
}
for _, result := range results {
@ -1755,17 +1818,17 @@ func repoStatsCheck(checker *repoChecker) {
log.Trace("Updating %s: %d", checker.desc, id)
_, err = x.Exec(checker.correctSQL, id, id)
if err != nil {
log.Error(4, "Update %s[%d]: %v", checker.desc, id, err)
log.Error(2, "Update %s[%d]: %v", checker.desc, id, err)
}
}
}
func CheckRepoStats() {
if taskStatusTable.IsRunning(_CHECK_REPOs) {
if taskStatusTable.IsRunning(_CHECK_REPO_STATS) {
return
}
taskStatusTable.Start(_CHECK_REPOs)
defer taskStatusTable.Stop(_CHECK_REPOs)
taskStatusTable.Start(_CHECK_REPO_STATS)
defer taskStatusTable.Stop(_CHECK_REPO_STATS)
log.Trace("Doing: CheckRepoStats")
@ -1809,14 +1872,14 @@ func CheckRepoStats() {
desc := "repository count 'num_closed_issues'"
results, err := x.Query("SELECT repo.id FROM `repository` repo WHERE repo.num_closed_issues!=(SELECT COUNT(*) FROM `issue` WHERE repo_id=repo.id AND is_closed=? AND is_pull=?)", true, false)
if err != nil {
log.Error(4, "Select %s: %v", desc, err)
log.Error(2, "Select %s: %v", desc, err)
} else {
for _, result := range results {
id := com.StrTo(result["id"]).MustInt64()
log.Trace("Updating %s: %d", desc, id)
_, err = x.Exec("UPDATE `repository` SET num_closed_issues=(SELECT COUNT(*) FROM `issue` WHERE repo_id=? AND is_closed=? AND is_pull=?) WHERE id=?", id, true, false, id)
if err != nil {
log.Error(4, "Update %s[%d]: %v", desc, id, err)
log.Error(2, "Update %s[%d]: %v", desc, id, err)
}
}
}
@ -1826,7 +1889,7 @@ func CheckRepoStats() {
// ***** START: Repository.NumForks *****
results, err = x.Query("SELECT repo.id FROM `repository` repo WHERE repo.num_forks!=(SELECT COUNT(*) FROM `repository` WHERE fork_id=repo.id)")
if err != nil {
log.Error(4, "Select repository count 'num_forks': %v", err)
log.Error(2, "Select repository count 'num_forks': %v", err)
} else {
for _, result := range results {
id := com.StrTo(result["id"]).MustInt64()
@ -1834,19 +1897,19 @@ func CheckRepoStats() {
repo, err := GetRepositoryByID(id)
if err != nil {
log.Error(4, "GetRepositoryByID[%d]: %v", id, err)
log.Error(2, "GetRepositoryByID[%d]: %v", id, err)
continue
}
rawResult, err := x.Query("SELECT COUNT(*) FROM `repository` WHERE fork_id=?", repo.ID)
if err != nil {
log.Error(4, "Select count of forks[%d]: %v", repo.ID, err)
log.Error(2, "Select count of forks[%d]: %v", repo.ID, err)
continue
}
repo.NumForks = int(parseCountResult(rawResult))
if err = UpdateRepository(repo, false); err != nil {
log.Error(4, "UpdateRepository[%d]: %v", id, err)
log.Error(2, "UpdateRepository[%d]: %v", id, err)
continue
}
}

4
modules/bindata/bindata.go
View File

File diff suppressed because one or more lines are too long

17
modules/cron/cron.go
Unescape View File

@ -25,7 +25,7 @@ func NewContext() {
if setting.Cron.UpdateMirror.Enabled {
entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, models.MirrorUpdate)
if err != nil {
log.Fatal(4, "Cron[Update mirrors]: %v", err)
log.Fatal(2, "Cron.(update mirrors): %v", err)
}
if setting.Cron.UpdateMirror.RunAtStart {
entry.Prev = time.Now()
@ -36,7 +36,7 @@ func NewContext() {
if setting.Cron.RepoHealthCheck.Enabled {
entry, err = c.AddFunc("Repository health check", setting.Cron.RepoHealthCheck.Schedule, models.GitFsck)
if err != nil {
log.Fatal(4, "Cron[Repository health check]: %v", err)
log.Fatal(2, "Cron.(repository health check): %v", err)
}
if setting.Cron.RepoHealthCheck.RunAtStart {
entry.Prev = time.Now()
@ -47,7 +47,7 @@ func NewContext() {
if setting.Cron.CheckRepoStats.Enabled {
entry, err = c.AddFunc("Check repository statistics", setting.Cron.CheckRepoStats.Schedule, models.CheckRepoStats)
if err != nil {
log.Fatal(4, "Cron[Check repository statistics]: %v", err)
log.Fatal(2, "Cron.(check repository statistics): %v", err)
}
if setting.Cron.CheckRepoStats.RunAtStart {
entry.Prev = time.Now()
@ -55,6 +55,17 @@ func NewContext() {
go models.CheckRepoStats()
}
}
if setting.Cron.RepoArchiveCleanup.Enabled {
entry, err = c.AddFunc("Repository archive cleanup", setting.Cron.RepoArchiveCleanup.Schedule, models.DeleteOldRepositoryArchives)
if err != nil {
log.Fatal(2, "Cron.(repository archive cleanup): %v", err)
}
if setting.Cron.RepoArchiveCleanup.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
go models.DeleteOldRepositoryArchives()
}
}
c.Start()
}

6
modules/setting/setting.go
Unescape View File

@ -208,6 +208,12 @@ var (
RunAtStart bool
Schedule string
} `ini:"cron.check_repo_stats"`
RepoArchiveCleanup struct {
Enabled bool
RunAtStart bool
Schedule string
OlderThan time.Duration
} `ini:"cron.repo_archive_cleanup"`
}
// Git settings

2
templates/.VERSION
Unescape View File

@ -1 +1 @@
0.9.158.0218
0.9.159.0218

2
vendor/github.com/gogits/git-module/repo_commit.go generated vendored
Unescape View File

@ -209,7 +209,7 @@ func (repo *Repository) CommitsByRangeSize(revision string, page, size int) (*li
return repo.parsePrettyFormatLogToList(stdout)
}
const DEFAULT_COMMITS_PAGE_SIZE = 50
const DEFAULT_COMMITS_PAGE_SIZE = 30
func (repo *Repository) CommitsByRange(revision string, page int) (*list.List, error) {
return repo.CommitsByRangeSize(revision, page, DEFAULT_COMMITS_PAGE_SIZE)

6
vendor/vendor.json vendored
Unescape View File

@ -159,10 +159,10 @@
"revisionTime": "2016-08-10T03:50:02Z"
},
{
"checksumSHA1": "mR45j8svu6CZu81VqN+lfgpCVjA=",
"checksumSHA1": "JPQWxQRYFpezMThuSqcPhvbJdq4=",
"path": "github.com/gogits/git-module",
"revision": "fa2ace85ecb113f89f6862d8a6e3075a7aa425b9",
"revisionTime": "2017-02-18T23:35:37Z"
"revision": "c882f3d24df5fb730d3b3c1b6cc64416dd22660c",
"revisionTime": "2017-02-18T23:44:51Z"
},
{
"checksumSHA1": "xvG+RgJODQqlmdAkHUQK2TyLR88=",

Write Preview
Loading…
Cancel
Save