playground
/
gogs
mirror of https://github.com/gogits/gogs.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

cmd: able to backup and restore 

Not very robust, must execute under correct workdir.

Addresses #2072, #3708, #648
pull/4240/head
Unknwon 8 years ago
parent
b06f299748
commit
ca2cfaf71e
No known key found for this signature in database
GPG Key ID: 25B575AE3213B2B3
7 changed files with 356 additions and 128 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 135
      cmd/backup.go
  2. 122
      cmd/dump.go
  3. 129
      cmd/restore.go
  4. 2
      cmd/web.go
  5. 3
      gogs.go
  6. 2
      models/migrations/migrations.go
  7. 91
      models/models.go

135
cmd/backup.go
Unescape View File

@ -0,0 +1,135 @@
// Copyright 2017 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"fmt"
"io/ioutil"
"os"
"path"
"time"
"github.com/Unknwon/cae/zip"
"github.com/Unknwon/com"
"github.com/urfave/cli"
log "gopkg.in/clog.v1"
"gopkg.in/ini.v1"
"github.com/gogits/gogs/models"
"github.com/gogits/gogs/modules/setting"
)
var Backup = cli.Command{
Name: "backup",
Usage: "Backup files and database",
Description: `Backup dumps and compresses all related files and database into zip file,
which can be used for migrating Gogs to another server. The output format is meant to be
portable among all supported database engines.`,
Action: runBackup,
Flags: []cli.Flag{
stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
boolFlag("verbose, v", "Show process details"),
stringFlag("tempdir, t", os.TempDir(), "Temporary directory path"),
stringFlag("target", "./", "Target directory path to save backup archive"),
boolFlag("database-only", "Only dump database"),
boolFlag("exclude-repos", "Exclude repositories"),
},
}
const _ARCHIVE_ROOT_DIR = "gogs-backup"
func runBackup(c *cli.Context) error {
zip.Verbose = c.Bool("verbose")
if c.IsSet("config") {
setting.CustomConf = c.String("config")
}
setting.NewContext()
models.LoadConfigs()
models.SetEngine()
tmpDir := c.String("tempdir")
if !com.IsExist(tmpDir) {
log.Fatal(0, "'--tempdir' does not exist: %s", tmpDir)
}
rootDir, err := ioutil.TempDir(tmpDir, "gogs-backup-")
if err != nil {
log.Fatal(0, "Fail to create backup root directory '%s': %v", rootDir, err)
}
log.Info("Backup root directory: %s", rootDir)
// Metadata
metaFile := path.Join(rootDir, "metadata.ini")
metadata := ini.Empty()
metadata.Section("").Key("VERSION").SetValue("1")
metadata.Section("").Key("DATE_TIME").SetValue(time.Now().String())
metadata.Section("").Key("GOGS_VERSION").SetValue(setting.AppVer)
if err = metadata.SaveTo(metaFile); err != nil {
log.Fatal(0, "Fail to save metadata '%s': %v", metaFile, err)
}
archiveName := path.Join(c.String("target"), fmt.Sprintf("gogs-backup-%d.zip", time.Now().Unix()))
log.Info("Packing backup files to: %s", archiveName)
z, err := zip.Create(archiveName)
if err != nil {
log.Fatal(0, "Fail to create backup archive '%s': %v", archiveName, err)
}
if err = z.AddFile(_ARCHIVE_ROOT_DIR+"/metadata.ini", metaFile); err != nil {
log.Fatal(0, "Fail to include 'metadata.ini': %v", err)
}
// Database
dbDir := path.Join(rootDir, "db")
if err = models.DumpDatabase(dbDir); err != nil {
log.Fatal(0, "Fail to dump database: %v", err)
}
if err = z.AddDir(_ARCHIVE_ROOT_DIR+"/db", dbDir); err != nil {
log.Fatal(0, "Fail to include 'db': %v", err)
}
// Custom files
if !c.Bool("database-only") {
if err = z.AddDir(_ARCHIVE_ROOT_DIR+"/custom", setting.CustomPath); err != nil {
log.Fatal(0, "Fail to include 'custom': %v", err)
}
}
// Data files
if !c.Bool("database-only") {
for _, dir := range []string{"attachments", "avatars"} {
dirPath := path.Join(setting.AppDataPath, dir)
if !com.IsDir(dirPath) {
continue
}
if err = z.AddDir(path.Join(_ARCHIVE_ROOT_DIR+"/data", dir), dirPath); err != nil {
log.Fatal(0, "Fail to include 'data': %v", err)
}
}
}
// Repositories
if !c.Bool("exclude-repos") && !c.Bool("database-only") {
reposDump := path.Join(rootDir, "repositories.zip")
log.Info("Dumping repositories in '%s'", setting.RepoRootPath)
if err = zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
log.Fatal(0, "Fail to dump repositories: %v", err)
}
log.Info("Repositories dumped to: %s", reposDump)
if err = z.AddFile(_ARCHIVE_ROOT_DIR+"/repositories.zip", reposDump); err != nil {
log.Fatal(0, "Fail to include 'repositories.zip': %v", err)
}
}
if err = z.Close(); err != nil {
log.Fatal(0, "Fail to save backup archive '%s': %v", archiveName, err)
}
os.RemoveAll(rootDir)
log.Info("Backup succeed! Archive is located at: %s", archiveName)
log.Shutdown()
return nil
}

122
cmd/dump.go
Unescape View File

@ -1,122 +0,0 @@
// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"fmt"
"log"
"os"
"path"
"time"
"io/ioutil"
"github.com/Unknwon/cae/zip"
"github.com/Unknwon/com"
"github.com/urfave/cli"
"github.com/gogits/gogs/models"
"github.com/gogits/gogs/modules/setting"
)
var Dump = cli.Command{
Name: "dump",
Usage: "Dump Gogs files and database",
Description: `Dump compresses all related files and database into zip file.
It can be used for backup and capture Gogs server image to send to maintainer`,
Action: runDump,
Flags: []cli.Flag{
stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
boolFlag("verbose, v", "Show process details"),
stringFlag("tempdir, t", os.TempDir(), "Temporary dir path"),
},
}
func runDump(ctx *cli.Context) error {
if ctx.IsSet("config") {
setting.CustomConf = ctx.String("config")
}
setting.NewContext()
models.LoadConfigs()
models.SetEngine()
tmpDir := ctx.String("tempdir")
if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
log.Fatalf("Path does not exist: %s", tmpDir)
}
TmpWorkDir, err := ioutil.TempDir(tmpDir, "gogs-dump-")
if err != nil {
log.Fatalf("Fail to create tmp work directory: %v", err)
}
log.Printf("Creating tmp work dir: %s", TmpWorkDir)
reposDump := path.Join(TmpWorkDir, "gogs-repo.zip")
dbDump := path.Join(TmpWorkDir, "gogs-db.sql")
log.Printf("Dumping local repositories...%s", setting.RepoRootPath)
zip.Verbose = ctx.Bool("verbose")
if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
log.Fatalf("Fail to dump local repositories: %v", err)
}
log.Printf("Dumping database...")
if err := models.DumpDatabase(dbDump); err != nil {
log.Fatalf("Fail to dump database: %v", err)
}
fileName := fmt.Sprintf("gogs-dump-%d.zip", time.Now().Unix())
log.Printf("Packing dump files...")
z, err := zip.Create(fileName)
if err != nil {
os.Remove(fileName)
log.Fatalf("Fail to create %s: %v", fileName, err)
}
if err := z.AddFile("gogs-repo.zip", reposDump); err != nil {
log.Fatalf("Fail to include gogs-repo.zip: %v", err)
}
if err := z.AddFile("gogs-db.sql", dbDump); err != nil {
log.Fatalf("Fail to include gogs-db.sql: %v", err)
}
customDir, err := os.Stat(setting.CustomPath)
if err == nil && customDir.IsDir() {
if err := z.AddDir("custom", setting.CustomPath); err != nil {
log.Fatalf("Fail to include custom: %v", err)
}
} else {
log.Printf("Custom dir %s doesn't exist, skipped", setting.CustomPath)
}
if err := z.AddDir("log", setting.LogRootPath); err != nil {
log.Fatalf("Fail to include log: %v", err)
}
for _, dir := range []string{"attachments", "avatars"} {
dirPath := path.Join(setting.AppDataPath, dir)
if !com.IsDir(dirPath) {
continue
}
if err := z.AddDir(path.Join("data", dir), dirPath); err != nil {
log.Fatalf("Fail to include '%s': %v", dirPath, err)
}
}
// FIXME: SSH key file.
if err = z.Close(); err != nil {
os.Remove(fileName)
log.Fatalf("Fail to save %s: %v", fileName, err)
}
if err := os.Chmod(fileName, 0600); err != nil {
log.Printf("Can't change file access permissions mask to 0600: %v", err)
}
log.Printf("Removing tmp work dir: %s", TmpWorkDir)
os.RemoveAll(TmpWorkDir)
log.Printf("Finish dumping in file %s", fileName)
return nil
}

129
cmd/restore.go
Unescape View File

@ -0,0 +1,129 @@
// Copyright 2017 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"os"
"path"
"github.com/Unknwon/cae/zip"
"github.com/Unknwon/com"
"github.com/mcuadros/go-version"
"github.com/urfave/cli"
log "gopkg.in/clog.v1"
"gopkg.in/ini.v1"
"github.com/gogits/gogs/models"
"github.com/gogits/gogs/modules/setting"
)
var Restore = cli.Command{
Name: "restore",
Usage: "Restore files and database from backup",
Description: `Restore imports all related files and database from a backup archive.
The backup version must lower or equal to current Gogs version. You can also import
backup from other database engines, which is useful for database migrating.
If corresponding files or database tables are not presented in the archive, they will
be skipped and remian unchanged.`,
Action: runRestore,
Flags: []cli.Flag{
stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
boolFlag("verbose, v", "Show process details"),
stringFlag("tempdir, t", os.TempDir(), "Temporary directory path"),
stringFlag("from", "", "Path to backup archive"),
boolFlag("database-only", "Only import database"),
boolFlag("exclude-repos", "Exclude repositories"),
},
}
func runRestore(c *cli.Context) error {
zip.Verbose = c.Bool("verbose")
tmpDir := c.String("tempdir")
if !com.IsExist(tmpDir) {
log.Fatal(0, "'--tempdir' does not exist: %s", tmpDir)
}
log.Info("Restore backup from: %s", c.String("from"))
if err := zip.ExtractTo(c.String("from"), tmpDir); err != nil {
log.Fatal(0, "Fail to extract backup archive: %v", err)
}
archivePath := path.Join(tmpDir, _ARCHIVE_ROOT_DIR)
// Check backup version
metaFile := path.Join(archivePath, "metadata.ini")
if !com.IsExist(metaFile) {
log.Fatal(0, "File 'metadata.ini' is missing")
}
metadata, err := ini.Load(metaFile)
if err != nil {
log.Fatal(0, "Fail to load metadata '%s': %v", metaFile, err)
}
backupVersion := metadata.Section("").Key("GOGS_VERSION").MustString("999.0")
if version.Compare(setting.AppVer, backupVersion, "<") {
log.Fatal(0, "Current Gogs version is lower than backup version: %s < %s", setting.AppVer, backupVersion)
}
// If config file is not present in backup, user must set this file via flag.
// Otherwise, it's optional to set config file flag.
configFile := path.Join(archivePath, "custom/conf/app.ini")
if c.IsSet("config") {
setting.CustomConf = c.String("config")
} else if !com.IsExist(configFile) {
log.Fatal(0, "'--config' is not specified and custom config file is not found in backup")
} else {
setting.CustomConf = configFile
}
setting.NewContext()
models.LoadConfigs()
models.SetEngine()
// Database
dbDir := path.Join(archivePath, "db")
if err = models.ImportDatabase(dbDir); err != nil {
log.Fatal(0, "Fail to import database: %v", err)
}
// Custom files
if !c.Bool("database-only") {
if com.IsExist(setting.CustomPath) {
if err = os.Rename(setting.CustomPath, setting.CustomPath+".bak"); err != nil {
log.Fatal(0, "Fail to backup current 'custom': %v", err)
}
}
if err = os.Rename(path.Join(archivePath, "custom"), setting.CustomPath); err != nil {
log.Fatal(0, "Fail to import 'custom': %v", err)
}
}
// Data files
if !c.Bool("database-only") {
for _, dir := range []string{"attachments", "avatars"} {
dirPath := path.Join(setting.AppDataPath, dir)
if com.IsExist(dirPath) {
if err = os.Rename(dirPath, dirPath+".bak"); err != nil {
log.Fatal(0, "Fail to backup current 'data': %v", err)
}
}
if err = os.Rename(path.Join(archivePath, "data", dir), dirPath); err != nil {
log.Fatal(0, "Fail to import 'data': %v", err)
}
}
}
// Repositories
reposPath := path.Join(archivePath, "repositories.zip")
if !c.Bool("exclude-repos") && !c.Bool("database-only") && com.IsExist(reposPath) {
if err := zip.ExtractTo(reposPath, path.Dir(setting.RepoRootPath)); err != nil {
log.Fatal(0, "Fail to extract 'repositories.zip': %v", err)
}
}
os.RemoveAll(path.Join(tmpDir, _ARCHIVE_ROOT_DIR))
log.Info("Restore succeed!")
log.Shutdown()
return nil
}

2
cmd/web.go
Unescape View File

@ -51,7 +51,7 @@ import (
var Web = cli.Command{
Name: "web",
Usage: "Start Gogs web server",
Usage: "Start web server",
Description: `Gogs web server is the only thing you need to run,
and it takes care of all the other things for you`,
Action: runWeb,

3
gogs.go
Unescape View File

@ -31,10 +31,11 @@ func main() {
cmd.Web,
cmd.Serv,
cmd.Hook,
cmd.Dump,
cmd.Cert,
cmd.Admin,
cmd.Import,
cmd.Backup,
cmd.Restore,
}
app.Flags = append(app.Flags, []cli.Flag{}...)
app.Run(os.Args)

2
models/migrations/migrations.go
Unescape View File

@ -42,7 +42,7 @@ func (m *migration) Migrate(x *xorm.Engine) error {
// The version table. Should have only one row with id==1
type Version struct {
ID int64 `xorm:"pk autoincr"`
ID int64
Version int64
}

91
models/models.go
Unescape View File

@ -5,7 +5,9 @@
package models
import (
"bufio"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/url"
@ -13,6 +15,7 @@ import (
"path"
"strings"
"github.com/Unknwon/com"
_ "github.com/denisenkom/go-mssqldb"
_ "github.com/go-sql-driver/mysql"
"github.com/go-xorm/core"
@ -262,7 +265,89 @@ func Ping() error {
return x.Ping()
}
// DumpDatabase dumps all data from database to file system.
func DumpDatabase(filePath string) error {
return x.DumpAllToFile(filePath)
// The version table. Should have only one row with id==1
type Version struct {
ID int64
Version int64
}
// DumpDatabase dumps all data from database to file system in JSON format.
func DumpDatabase(dirPath string) (err error) {
os.MkdirAll(dirPath, os.ModePerm)
// Purposely create a local variable to not modify global variable
tables := append(tables, new(Version))
for _, table := range tables {
tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*models.")
tableFile := path.Join(dirPath, tableName+".json")
f, err := os.Create(tableFile)
if err != nil {
return fmt.Errorf("fail to create JSON file: %v", err)
}
if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) {
enc := json.NewEncoder(f)
return enc.Encode(bean)
}); err != nil {
f.Close()
return fmt.Errorf("fail to dump table '%s': %v", tableName, err)
}
f.Close()
}
return nil
}
// ImportDatabase imports data from backup archive.
func ImportDatabase(dirPath string) (err error) {
// Purposely create a local variable to not modify global variable
tables := append(tables, new(Version))
for _, table := range tables {
tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*models.")
tableFile := path.Join(dirPath, tableName+".json")
if !com.IsExist(tableFile) {
continue
}
if err = x.DropTables(table); err != nil {
return fmt.Errorf("fail to drop table '%s': %v", tableName, err)
} else if err = x.Sync2(table); err != nil {
return fmt.Errorf("fail to sync table '%s': %v", tableName, err)
}
f, err := os.Open(tableFile)
if err != nil {
return fmt.Errorf("fail to open JSON file: %v", err)
}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
switch bean := table.(type) {
case *LoginSource:
meta := make(map[string]interface{})
if err = json.Unmarshal(scanner.Bytes(), &meta); err != nil {
return fmt.Errorf("fail to unmarshal to map: %v", err)
}
tp := LoginType(com.StrTo(com.ToStr(meta["Type"])).MustInt64())
switch tp {
case LOGIN_LDAP, LOGIN_DLDAP:
bean.Cfg = new(LDAPConfig)
case LOGIN_SMTP:
bean.Cfg = new(SMTPConfig)
case LOGIN_PAM:
bean.Cfg = new(PAMConfig)
default:
return fmt.Errorf("unrecognized login source type:: %v", tp)
}
table = bean
}
if err = json.Unmarshal(scanner.Bytes(), table); err != nil {
return fmt.Errorf("fail to unmarshal to struct: %v", err)
}
if _, err = x.Insert(table); err != nil {
return fmt.Errorf("fail to insert strcut: %v", err)
}
}
}
return nil
}

Write Preview
Loading…
Cancel
Save