diff --git a/cmd/get.go b/cmd/get.go index 8d59d8299..b73623e98 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -160,6 +160,7 @@ func getByPath(ctx *cli.Context) { nodes = append(nodes, node) } + doc.LoadLocalNodes() downloadPackages(ctx, nodes) if doc.LocalNodes != nil { @@ -197,7 +198,7 @@ func downloadPackages(ctx *cli.Context, nodes []*doc.Node) { if !downloadCache[n.ImportPath] { // Download package. - nod, imports := downloadPackage(n) + nod, imports := downloadPackage(ctx, n) if len(imports) > 0 { var gf *goconfig.ConfigFile @@ -242,8 +243,8 @@ func downloadPackages(ctx *cli.Context, nodes []*doc.Node) { downloadCount++ // Only save non-commit node. - if len(nod.Value) == 0 { - doc.SaveNode(nod) + if len(nod.Value) == 0 && len(nod.Revision) > 0 { + doc.LocalNodes.SetValue(nod.ImportPath, "value", nod.Revision) } } } else { @@ -262,13 +263,14 @@ func downloadPackages(ctx *cli.Context, nodes []*doc.Node) { } // downloadPackage downloads package either use version control tools or not. -func downloadPackage(nod *doc.Node) (*doc.Node, []string) { +func downloadPackage(ctx *cli.Context, nod *doc.Node) (*doc.Node, []string) { log.Message("Downloading", fmt.Sprintf("package: %s@%s:%s", nod.ImportPath, nod.Type, doc.CheckNodeValue(nod.Value))) // Mark as donwloaded. downloadCache[nod.ImportPath] = true - imports, err := doc.PureDownload(nod, installRepoPath, nil) //CmdGet.Flags) + nod.Revision = doc.LocalNodes.MustValue(nod.ImportPath, "value") + imports, err := doc.PureDownload(nod, installRepoPath, ctx) //CmdGet.Flags) if err != nil { log.Error("Get", "Fail to download pakage: "+nod.ImportPath) diff --git a/doc/bitbucket.go b/doc/bitbucket.go index 0b192a9ba..b5970ef91 100644 --- a/doc/bitbucket.go +++ b/doc/bitbucket.go @@ -27,6 +27,9 @@ import ( "strings" "github.com/Unknwon/com" + "github.com/codegangsta/cli" + + "github.com/gpmgo/gopm/log" ) var ( @@ -35,7 +38,7 @@ var ( ) // getBitbucketDoc downloads tarball from bitbucket.org. -func getBitbucketDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { +func getBitbucketDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, ctx *cli.Context) ([]string, error) { // Check version control. if m := bitbucketEtagRe.FindStringSubmatch(nod.Value); m != nil { match["vcs"] = m[1] @@ -152,7 +155,7 @@ func getBitbucketDoc(client *http.Client, match map[string]string, installRepoPa // Create diretory before create file. dir := path.Dir(absPath) - if !checkDir(dir, dirs) && !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) { + if !checkDir(dir, dirs) && !(!ctx.Bool("example") && strings.Contains(absPath, "example")) { dirs = append(dirs, dir) os.MkdirAll(dir+"/", os.ModePerm) } diff --git a/doc/conf.go b/doc/conf.go index bc20bf157..a20e00f28 100644 --- a/doc/conf.go +++ b/doc/conf.go @@ -72,23 +72,19 @@ func LoadPkgNameList(filePath string) { } } -func SaveNode(nod *Node) { - if LocalNodes == nil { - if !com.IsDir(HomeDir + "/data") { - os.Mkdir(HomeDir+"/data", os.ModePerm) - } - - if !com.IsFile(HomeDir + LocalNodesFile) { - os.Create(HomeDir + LocalNodesFile) - } +func LoadLocalNodes() { + if !com.IsDir(HomeDir + "/data") { + os.MkdirAll(HomeDir+"/data", os.ModePerm) + } - var err error - LocalNodes, err = goconfig.LoadConfigFile(HomeDir + LocalNodesFile) - if err != nil { - log.Error("Save node", "Fail to load localnodes.list") - log.Fatal("", err.Error()) - } + if !com.IsFile(HomeDir + LocalNodesFile) { + os.Create(HomeDir + LocalNodesFile) } - LocalNodes.SetValue(nod.ImportPath, "value", nod.Value) + var err error + LocalNodes, err = goconfig.LoadConfigFile(HomeDir + LocalNodesFile) + if err != nil { + log.Error("Load node", "Fail to load localnodes.list") + log.Fatal("", err.Error()) + } } diff --git a/doc/github.go b/doc/github.go index a47c39a94..c7885cbd3 100644 --- a/doc/github.go +++ b/doc/github.go @@ -26,6 +26,9 @@ import ( "strings" "github.com/Unknwon/com" + "github.com/codegangsta/cli" + + "github.com/gpmgo/gopm/log" ) var ( @@ -33,12 +36,46 @@ var ( ) // getGithubDoc downloads tarball from github.com. -func getGithubDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { +func getGithubDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, ctx *cli.Context) ([]string, error) { // Check downlaod type. switch nod.Type { case BRANCH: if len(nod.Value) == 0 { match["sha"] = MASTER + + // Only get and check revision with the latest version. + var refs []*struct { + Ref string + Url string + Object struct { + Sha string + Type string + Url string + } + } + + err := com.HttpGetJSON(client, com.Expand("https://api.github.com/repos/{owner}/{repo}/git/refs?{cred}", match), &refs) + if err != nil { + log.Error("GET", "Fail to get revision") + log.Error("", err.Error()) + break + } + + var etag string + COMMIT_LOOP: + for _, ref := range refs { + switch { + case strings.HasPrefix(ref.Ref, "refs/heads/master"): + etag = ref.Object.Sha + break COMMIT_LOOP + } + } + if etag == nod.Revision { + log.Log("GET Package hasn't changed: %s", nod.ImportPath) + return nil, nil + } + nod.Revision = etag + } else { match["sha"] = nod.Value } @@ -97,7 +134,7 @@ func getGithubDoc(client *http.Client, match map[string]string, installRepoPath switch { case strings.HasSuffix(absPath, "/"): // Directory. // Check if current directory is example. - if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) { + if !(!ctx.Bool("example") && strings.Contains(absPath, "example")) { for _, d := range dirs { if d == absPath { break compareDir diff --git a/doc/google.go b/doc/google.go index ee2b60e39..404c33a41 100644 --- a/doc/google.go +++ b/doc/google.go @@ -26,17 +26,21 @@ import ( "strings" "github.com/Unknwon/com" + "github.com/codegangsta/cli" + + "github.com/gpmgo/gopm/log" ) var ( - googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`) - googleFileRe = regexp.MustCompile(`
  • [a-z0-9\-]+)(:?\.(?P[a-z0-9\-]+))?(?P/[a-z0-9A-Z_.\-/]+)?$`) + googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`) + googleRevisionRe = regexp.MustCompile(`

    (?:[^ ]+ - )?Revision *([^:]+):`) + googleFileRe = regexp.MustCompile(`
  • [a-z0-9\-]+)(:?\.(?P[a-z0-9\-]+))?(?P/[a-z0-9A-Z_.\-/]+)?$`) ) // getGoogleDoc downloads raw files from code.google.com. -func getGoogleDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { +func getGoogleDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, ctx *cli.Context) ([]string, error) { setupGoogleMatch(match) // Check version control. if err := getGoogleVCS(client, match); err != nil { @@ -47,6 +51,28 @@ func getGoogleDoc(client *http.Client, match map[string]string, installRepoPath case BRANCH: if len(nod.Value) == 0 { match["tag"] = defaultTags[match["vcs"]] + + // Only get and check revision with the latest version. + p, err := com.HttpGetBytes(client, + com.Expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/?r={tag}", match), nil) + if err != nil { + log.Error("GET", "Fail to get revision") + log.Error("", err.Error()) + break + } + + if m := googleRevisionRe.FindSubmatch(p); m == nil { + log.Error("GET", "Fail to get revision") + log.Error("", err.Error()) + } else { + etag := string(m[1]) + if etag == nod.Revision { + log.Log("GET Package hasn't changed: %s", nod.ImportPath) + return nil, nil + } + nod.Revision = etag + } + } else { match["tag"] = nod.Value } @@ -106,7 +132,7 @@ func getGoogleDoc(client *http.Client, match map[string]string, installRepoPath // Create diretory before create file. dir := path.Dir(absPath) - if !checkDir(dir, dirs) && !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) { + if !checkDir(dir, dirs) && !(!ctx.Bool("example") && strings.Contains(absPath, "example")) { dirs = append(dirs, dir+"/") os.MkdirAll(dir+"/", os.ModePerm) } diff --git a/doc/launchpad.go b/doc/launchpad.go index 24b291d54..9cb535fd7 100644 --- a/doc/launchpad.go +++ b/doc/launchpad.go @@ -25,12 +25,13 @@ import ( "strings" "github.com/Unknwon/com" + "github.com/codegangsta/cli" ) var launchpadPattern = regexp.MustCompile(`^launchpad\.net/(?P(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]+)*$`) // getLaunchpadDoc downloads tarball from launchpad.net. -func getLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { +func getLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, ctx *cli.Context) ([]string, error) { if match["project"] != "" && match["series"] != "" { rc, err := com.HttpGet(client, com.Expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil) @@ -99,7 +100,7 @@ func getLaunchpadDoc(client *http.Client, match map[string]string, installRepoPa // Create diretory before create file. os.MkdirAll(absPath+"/", os.ModePerm) // Check if current directory is example. - if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) { + if !(!ctx.Bool("example") && strings.Contains(absPath, "example")) { dirs = append(dirs, absPath) } case !strings.HasPrefix(fn, "."): diff --git a/doc/oschina.go b/doc/oschina.go index ab3777c3a..b35151030 100644 --- a/doc/oschina.go +++ b/doc/oschina.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/Unknwon/com" + "github.com/codegangsta/cli" ) var ( @@ -33,7 +34,7 @@ var ( ) // getGithubDoc downloads tarball from git.oschina.com. -func getOSCDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) { +func getOSCDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, ctx *cli.Context) ([]string, error) { // Check downlaod type. switch nod.Type { case BRANCH: diff --git a/doc/struct.go b/doc/struct.go index 94ca77b16..f1ad2d4b4 100644 --- a/doc/struct.go +++ b/doc/struct.go @@ -21,6 +21,8 @@ import ( "os" "regexp" "time" + + "github.com/codegangsta/cli" ) const ( @@ -58,6 +60,7 @@ type Node struct { DownloadURL string Synopsis string IsGetDeps bool + Revision string } func NewNode(importPath, downloadUrl, tp, value string, isGetDeps bool) *Node { @@ -96,7 +99,7 @@ type walker struct { type service struct { pattern *regexp.Regexp prefix string - get func(*http.Client, map[string]string, string, *Node, map[string]bool) ([]string, error) + get func(*http.Client, map[string]string, string, *Node, *cli.Context) ([]string, error) } // services is the list of source code control services handled by gopkgdoc. diff --git a/doc/vcs.go b/doc/vcs.go index a7ab3288a..58635db3b 100644 --- a/doc/vcs.go +++ b/doc/vcs.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/Unknwon/com" + "github.com/codegangsta/cli" ) var ( @@ -97,7 +98,7 @@ func bestTag(tags map[string]string, defaultTag string) (string, string, error) } // PureDownload downloads package without version control. -func PureDownload(nod *Node, installRepoPath string, flags map[string]bool) ([]string, error) { +func PureDownload(nod *Node, installRepoPath string, ctx *cli.Context) ([]string, error) { for _, s := range services { if s.get == nil || !strings.HasPrefix(nod.DownloadURL, s.prefix) { continue @@ -115,14 +116,14 @@ func PureDownload(nod *Node, installRepoPath string, flags map[string]bool) ([]s match[n] = m[i] } } - return s.get(HttpClient, match, installRepoPath, nod, flags) + return s.get(HttpClient, match, installRepoPath, nod, ctx) } com.ColorLog("[TRAC] Cannot match any service, getting dynamic...\n") - return getDynamic(HttpClient, nod, installRepoPath, flags) + return getDynamic(HttpClient, nod, installRepoPath, ctx) } -func getDynamic(client *http.Client, nod *Node, installRepoPath string, flags map[string]bool) ([]string, error) { +func getDynamic(client *http.Client, nod *Node, installRepoPath string, ctx *cli.Context) ([]string, error) { match, err := fetchMeta(client, nod.ImportPath) if err != nil { return nil, err @@ -139,7 +140,7 @@ func getDynamic(client *http.Client, nod *Node, installRepoPath string, flags ma } nod.DownloadURL = com.Expand("{repo}{dir}", match) - return PureDownload(nod, installRepoPath, flags) + return PureDownload(nod, installRepoPath, ctx) } func fetchMeta(client *http.Client, importPath string) (map[string]string, error) { diff --git a/log/log_nonwindows.go b/log/log.go similarity index 98% rename from log/log_nonwindows.go rename to log/log.go index f32490655..9c445ae61 100644 --- a/log/log_nonwindows.go +++ b/log/log.go @@ -12,6 +12,8 @@ // License for the specific language governing permissions and limitations // under the License. +// +build !windows + package log import (