From 352a57e88d595b19b071258291f9e8a7ee5ce584 Mon Sep 17 00:00:00 2001 From: Unknown Date: Sat, 18 May 2013 21:09:44 -0400 Subject: [PATCH] what the fuck is wrong??? install. --- build.go | 33 +- {i18n => conf}/gpm.toml | 0 doc/error.go | 37 ++ doc/github.go | 138 +++++++ doc/http.go | 143 +++++++ doc/struct.go | 35 ++ doc/vcs.go | 178 +++++++++ doc/walker.go | 149 +++++++ gpm.go | 39 +- i18n/{usage_en-US.tpl => en-US/usage.tpl} | 0 .../usage_build.txt} | 0 i18n/en-US/usage_install.txt | 23 ++ install.go | 169 ++++++++ models/models.go | 184 +++++++++ utils/utils.go | 368 ++++++++++++++++++ utils/utils_test.go | 24 ++ 16 files changed, 1485 insertions(+), 35 deletions(-) rename {i18n => conf}/gpm.toml (100%) create mode 100644 doc/error.go create mode 100644 doc/github.go create mode 100644 doc/http.go create mode 100644 doc/struct.go create mode 100644 doc/vcs.go create mode 100644 doc/walker.go rename i18n/{usage_en-US.tpl => en-US/usage.tpl} (100%) rename i18n/{usage_build_en-US.txt => en-US/usage_build.txt} (100%) create mode 100644 i18n/en-US/usage_install.txt create mode 100644 install.go create mode 100644 models/models.go create mode 100644 utils/utils_test.go diff --git a/build.go b/build.go index c9aa3e249..afd196f41 100644 --- a/build.go +++ b/build.go @@ -6,9 +6,7 @@ package main import ( "fmt" - "io" "os" - "os/exec" "path" "runtime" "strings" @@ -22,7 +20,6 @@ var cmdBuild = &Command{ func init() { cmdBuild.Run = runBuild - cmdBuild.Flags = []string{"-v"} } func runBuild(cmd *Command, args []string) { @@ -37,40 +34,24 @@ func runBuild(cmd *Command, args []string) { proName += ".exe" } - cmdExec := exec.Command("go", cmdArgs...) - stdout, err := cmdExec.StdoutPipe() - if err != nil { - fmt.Println(err) - } - stderr, err := cmdExec.StderrPipe() - if err != nil { - fmt.Println(err) - } - err = cmdExec.Start() - if err != nil { - fmt.Println(err) - } - go io.Copy(os.Stdout, stdout) - go io.Copy(os.Stderr, stderr) - cmdExec.Wait() + executeGoCommand(cmdArgs) // Find executable in GOPATH and copy to current directory. - gopath := strings.Replace(os.Getenv("GOPATH"), ";", ":", -1) - gopath = strings.Replace(gopath, "\\", "/", -1) - paths := strings.Split(gopath, ":") + paths := utils.GetGOPATH() + for _, v := range paths { if utils.IsExist(v + "/bin/" + proName) { - err = os.Remove(wd + "/" + proName) + err := os.Remove(wd + "/" + proName) if err != nil { - fmt.Println("Fail to remove file in current directory :", err) + fmt.Printf("Fail to remove file in current directory: %s.\n", err) return } err = os.Rename(v+"/bin/"+proName, wd+"/"+proName) if err == nil { - fmt.Println("Moved file from $GOPATH to current directory.") + fmt.Printf("Moved file from $GOPATH(%s) to current directory(%s).\n", v, wd) return } else { - fmt.Println("Fail to move file from $GOPATH to current directory :", err) + fmt.Printf("Fail to move file from $GOPATH(%s) to current directory: %s.\n", v, err) } break } diff --git a/i18n/gpm.toml b/conf/gpm.toml similarity index 100% rename from i18n/gpm.toml rename to conf/gpm.toml diff --git a/doc/error.go b/doc/error.go new file mode 100644 index 000000000..379d30a45 --- /dev/null +++ b/doc/error.go @@ -0,0 +1,37 @@ +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "errors" +) + +var ( + errNotModified = errors.New("package not modified") + ErrNoMatch = errors.New("no match") + errUpdateTimeout = errors.New("update timeout") +) + +type NotFoundError struct { + Message string +} + +func (e NotFoundError) Error() string { + return e.Message +} + +func isNotFound(err error) bool { + _, ok := err.(NotFoundError) + return ok +} + +type RemoteError struct { + Host string + err error +} + +func (e *RemoteError) Error() string { + return e.err.Error() +} diff --git a/doc/github.go b/doc/github.go new file mode 100644 index 000000000..b911a8e1b --- /dev/null +++ b/doc/github.go @@ -0,0 +1,138 @@ +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "net/http" + "os" + "path" + "regexp" + "strings" + + "github.com/GPMGo/gpm/models" + "github.com/GPMGo/gpm/utils" +) + +var ( + githubRawHeader = http.Header{"Accept": {"application/vnd.github-blob.raw"}} + GithubPattern = regexp.MustCompile(`^github\.com/(?P[a-z0-9A-Z_.\-]+)/(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]*)?$`) + githubCred string +) + +func SetGithubCredentials(id, secret string) { + githubCred = "client_id=" + id + "&client_secret=" + secret +} + +func GetGithubDoc(client *http.Client, match map[string]string, commit string) (*models.PkgInfo, error) { + SetGithubCredentials("1862bcb265171f37f36c", "308d71ab53ccd858416cfceaed52d5d5b7d53c5f") + match["cred"] = githubCred + + var refs []*struct { + Object struct { + Type string + Sha string + Url string + } + Ref string + Url string + } + + // Check if has specific commit. + if len(commit) == 0 { + // Get up-to-date version. + err := httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/git/refs?{cred}", match), &refs) + if err != nil { + return nil, err + } + + tags := make(map[string]string) + for _, ref := range refs { + switch { + case strings.HasPrefix(ref.Ref, "refs/heads/"): + tags[ref.Ref[len("refs/heads/"):]] = ref.Object.Sha + case strings.HasPrefix(ref.Ref, "refs/tags/"): + tags[ref.Ref[len("refs/tags/"):]] = ref.Object.Sha + } + } + + // Check revision tag. + match["tag"], commit, err = bestTag(tags, "master") + if err != nil { + return nil, err + } + } + + match["sha"] = commit + // Download zip. + p, err := httpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil) + if err != nil { + return nil, err + } + + r, err := zip.NewReader(bytes.NewReader(p), int64(len(p))) + if err != nil { + return nil, err + } + //defer r.Close() + + shaName := expand("{repo}-{sha}", match) + paths := utils.GetGOPATH() + importPath := "github.com/" + expand("{owner}/{repo}", match) + installPath := paths[0] + "/src/" + importPath + // Create destination directory + os.Mkdir(installPath, os.ModePerm) + + files := make([]*source, 0, len(r.File)) + for _, f := range r.File { + srcName := f.FileInfo().Name()[strings.Index(f.FileInfo().Name(), "/")+1:] + fmt.Printf("Unzipping %s...", srcName) + fn := strings.Replace(f.FileInfo().Name(), shaName, installPath, 1) + + // Get files from archive + rc, err := f.Open() + if err != nil { + return nil, err + } + + // Create diretory before create file + os.MkdirAll(path.Dir(fn), os.ModePerm) + // Write data to file + fw, _ := os.Create(fn) + if err != nil { + return nil, err + } + + _, err = io.Copy(fw, rc) + if err != nil { + return nil, err + } + + localF, _ := os.Open(fn) + fbytes := make([]byte, f.FileInfo().Size()) + n, _ := localF.Read(fbytes) + fmt.Println(n) + + // Check if Go source file. + if n > 0 && strings.HasSuffix(fn, ".go") { + files = append(files, &source{ + name: srcName, + data: fbytes, + }) + } + } + + w := &walker{ + pinfo: &models.PkgInfo{ + Path: importPath, + Commit: commit, + }, + } + + return w.build(files) +} diff --git a/doc/http.go b/doc/http.go new file mode 100644 index 000000000..5b9494e34 --- /dev/null +++ b/doc/http.go @@ -0,0 +1,143 @@ +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "time" +) + +var userAgent = "go application" + +var ( + dialTimeout = flag.Duration("dial_timeout", 5*time.Second, "Timeout for dialing an HTTP connection.") + readTimeout = flag.Duration("read_timeout", 10*time.Second, "Timeoout for reading an HTTP response.") + writeTimeout = flag.Duration("write_timeout", 5*time.Second, "Timeout writing an HTTP request.") +) + +type timeoutConn struct { + net.Conn +} + +func (c *timeoutConn) Read(p []byte) (int, error) { + return c.Conn.Read(p) +} + +func (c *timeoutConn) Write(p []byte) (int, error) { + // Reset timeouts when writing a request. + c.Conn.SetWriteDeadline(time.Now().Add(*readTimeout)) + c.Conn.SetWriteDeadline(time.Now().Add(*writeTimeout)) + return c.Conn.Write(p) +} +func timeoutDial(network, addr string) (net.Conn, error) { + c, err := net.DialTimeout(network, addr, *dialTimeout) + if err != nil { + return nil, err + } + return &timeoutConn{Conn: c}, nil +} + +var ( + httpTransport = &http.Transport{Dial: timeoutDial} + HttpClient = &http.Client{Transport: httpTransport} +) + +// httpGet gets the specified resource. ErrNotFound is returned if the server +// responds with status 404. +func httpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) { + rc, err := httpGet(client, url, header) + if err != nil { + return nil, err + } + p, err := ioutil.ReadAll(rc) + rc.Close() + return p, err +} + +// httpGet gets the specified resource. ErrNotFound is returned if the +// server responds with status 404. +func httpGet(client *http.Client, url string, header http.Header) (io.ReadCloser, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + for k, vs := range header { + req.Header[k] = vs + } + resp, err := client.Do(req) + if err != nil { + return nil, &RemoteError{req.URL.Host, err} + } + if resp.StatusCode == 200 { + return resp.Body, nil + } + resp.Body.Close() + if resp.StatusCode == 404 { // 403 can be rate limit error. || resp.StatusCode == 403 { + err = NotFoundError{"Resource not found: " + url} + } else { + err = &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", url, resp.StatusCode)} + } + return nil, err +} + +// fetchFiles fetches the source files specified by the rawURL field in parallel. +func fetchFiles(client *http.Client, files []*source, header http.Header) error { + ch := make(chan error, len(files)) + for i := range files { + go func(i int) { + req, err := http.NewRequest("GET", files[i].rawURL, nil) + if err != nil { + ch <- err + return + } + req.Header.Set("User-Agent", userAgent) + for k, vs := range header { + req.Header[k] = vs + } + resp, err := client.Do(req) + if err != nil { + ch <- &RemoteError{req.URL.Host, err} + return + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + ch <- &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", req.URL, resp.StatusCode)} + return + } + files[i].data, err = ioutil.ReadAll(resp.Body) + if err != nil { + ch <- &RemoteError{req.URL.Host, err} + return + } + ch <- nil + }(i) + } + for _ = range files { + if err := <-ch; err != nil { + return err + } + } + return nil +} + +func httpGetJSON(client *http.Client, url string, v interface{}) error { + rc, err := httpGet(client, url, nil) + if err != nil { + return err + } + defer rc.Close() + err = json.NewDecoder(rc).Decode(v) + if _, ok := err.(*json.SyntaxError); ok { + err = NotFoundError{"JSON syntax error at " + url} + } + return err +} diff --git a/doc/struct.go b/doc/struct.go new file mode 100644 index 000000000..85ce42714 --- /dev/null +++ b/doc/struct.go @@ -0,0 +1,35 @@ +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "go/token" + "os" + "time" + + "github.com/GPMGo/gpm/models" +) + +// source is source code file. +type source struct { + name string + browseURL string + rawURL string + data []byte +} + +func (s *source) Name() string { return s.name } +func (s *source) Size() int64 { return int64(len(s.data)) } +func (s *source) Mode() os.FileMode { return 0 } +func (s *source) ModTime() time.Time { return time.Time{} } +func (s *source) IsDir() bool { return false } +func (s *source) Sys() interface{} { return nil } + +// walker holds the state used when building the documentation. +type walker struct { + pinfo *models.PkgInfo + srcs map[string]*source // Source files. + fset *token.FileSet +} diff --git a/doc/vcs.go b/doc/vcs.go new file mode 100644 index 000000000..3d6541f40 --- /dev/null +++ b/doc/vcs.go @@ -0,0 +1,178 @@ +// Copyright 2012 Gary Burd +// +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "bytes" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "regexp" + "strconv" + "strings" +) + +// TODO: specify with command line flag +const repoRoot = "/tmp/gddo" + +var urlTemplates = []struct { + re *regexp.Regexp + template string + lineFmt string +}{ + { + regexp.MustCompile(`^git\.gitorious\.org/(?P[^/]+/[^/]+)$`), + "https://gitorious.org/{repo}/blobs/{tag}/{dir}{0}", + "#line%d", + }, + { + regexp.MustCompile(`^camlistore\.org/r/p/(?P[^/]+)$`), + "http://camlistore.org/code/?p={repo}.git;hb={tag};f={dir}{0}", + "#l%d", + }, +} + +// lookupURLTemplate finds an expand() template, match map and line number +// format for well known repositories. +func lookupURLTemplate(repo, dir, tag string) (string, map[string]string, string) { + if strings.HasPrefix(dir, "/") { + dir = dir[1:] + "/" + } + for _, t := range urlTemplates { + if m := t.re.FindStringSubmatch(repo); m != nil { + match := map[string]string{ + "dir": dir, + "tag": tag, + } + for i, name := range t.re.SubexpNames() { + if name != "" { + match[name] = m[i] + } + } + return t.template, match, t.lineFmt + } + } + return "", nil, "" +} + +type vcsCmd struct { + schemes []string + download func([]string, string, string) (string, string, error) +} + +var vcsCmds = map[string]*vcsCmd{ + "git": &vcsCmd{ + schemes: []string{"http", "https", "git"}, + download: downloadGit, + }, +} + +var lsremoteRe = regexp.MustCompile(`(?m)^([0-9a-f]{40})\s+refs/(?:tags|heads)/(.+)$`) + +func downloadGit(schemes []string, repo, savedEtag string) (string, string, error) { + var p []byte + var scheme string + for i := range schemes { + cmd := exec.Command("git", "ls-remote", "--heads", "--tags", schemes[i]+"://"+repo+".git") + log.Println(strings.Join(cmd.Args, " ")) + var err error + p, err = cmd.Output() + if err == nil { + scheme = schemes[i] + break + } + } + + if scheme == "" { + return "", "", NotFoundError{"VCS not found"} + } + + tags := make(map[string]string) + for _, m := range lsremoteRe.FindAllSubmatch(p, -1) { + tags[string(m[2])] = string(m[1]) + } + + tag, commit, err := bestTag(tags, "master") + if err != nil { + return "", "", err + } + + etag := scheme + "-" + commit + + if etag == savedEtag { + return "", "", errNotModified + } + + dir := path.Join(repoRoot, repo+".git") + p, err = ioutil.ReadFile(path.Join(dir, ".git/HEAD")) + switch { + case err != nil: + if err := os.MkdirAll(dir, 0777); err != nil { + return "", "", err + } + cmd := exec.Command("git", "clone", scheme+"://"+repo, dir) + log.Println(strings.Join(cmd.Args, " ")) + if err := cmd.Run(); err != nil { + return "", "", err + } + case string(bytes.TrimRight(p, "\n")) == commit: + return tag, etag, nil + default: + cmd := exec.Command("git", "fetch") + log.Println(strings.Join(cmd.Args, " ")) + cmd.Dir = dir + if err := cmd.Run(); err != nil { + return "", "", err + } + } + + cmd := exec.Command("git", "checkout", "--detach", "--force", commit) + cmd.Dir = dir + if err := cmd.Run(); err != nil { + return "", "", err + } + + return tag, etag, nil +} + +var defaultTags = map[string]string{"git": "master", "hg": "default"} + +func bestTag(tags map[string]string, defaultTag string) (string, string, error) { + if commit, ok := tags["go1"]; ok { + return "go1", commit, nil + } + if commit, ok := tags[defaultTag]; ok { + return defaultTag, commit, nil + } + return "", "", NotFoundError{"Tag or branch not found."} +} + +// expand replaces {k} in template with match[k] or subs[atoi(k)] if k is not in match. +func expand(template string, match map[string]string, subs ...string) string { + var p []byte + var i int + for { + i = strings.Index(template, "{") + if i < 0 { + break + } + p = append(p, template[:i]...) + template = template[i+1:] + i = strings.Index(template, "}") + if s, ok := match[template[:i]]; ok { + p = append(p, s...) + } else { + j, _ := strconv.Atoi(template[:i]) + p = append(p, subs[j]...) + } + template = template[i+1:] + } + p = append(p, template...) + return string(p) +} diff --git a/doc/walker.go b/doc/walker.go new file mode 100644 index 000000000..aebb8e317 --- /dev/null +++ b/doc/walker.go @@ -0,0 +1,149 @@ +// Copyright 2011 Gary Burd +// Copyright 2013 Unknown +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package doc + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "io" + "io/ioutil" + "os" + "path" + "regexp" + "runtime" + "strings" + "time" + + "github.com/GPMGo/gpm/models" +) + +type sliceWriter struct{ p *[]byte } + +func (w sliceWriter) Write(p []byte) (int, error) { + *w.p = append(*w.p, p...) + return len(p), nil +} + +func (w *walker) readDir(dir string) ([]os.FileInfo, error) { + if dir != w.pinfo.Path { + panic("unexpected") + } + fis := make([]os.FileInfo, 0, len(w.srcs)) + for _, src := range w.srcs { + fis = append(fis, src) + } + return fis, nil +} + +func (w *walker) openFile(path string) (io.ReadCloser, error) { + if strings.HasPrefix(path, w.pinfo.Path+"/") { + if src, ok := w.srcs[path[len(w.pinfo.Path)+1:]]; ok { + return ioutil.NopCloser(bytes.NewReader(src.data)), nil + } + } + panic("unexpected") +} + +func simpleImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) { + pkg := imports[path] + if pkg == nil { + // Guess the package name without importing it. Start with the last + // element of the path. + name := path[strings.LastIndex(path, "/")+1:] + + // Trim commonly used prefixes and suffixes containing illegal name + // runes. + name = strings.TrimSuffix(name, ".go") + name = strings.TrimSuffix(name, "-go") + name = strings.TrimPrefix(name, "go.") + name = strings.TrimPrefix(name, "go-") + name = strings.TrimPrefix(name, "biogo.") + + // It's also common for the last element of the path to contain an + // extra "go" prefix, but not always. TODO: examine unresolved ids to + // detect when trimming the "go" prefix is appropriate. + + pkg = ast.NewObj(ast.Pkg, name) + pkg.Data = ast.NewScope(nil) + imports[path] = pkg + } + return pkg, nil +} + +var buildPicPattern = regexp.MustCompile(`\[+!+\[+([a-zA-Z ]*)+\]+\(+[a-zA-z]+://[^\s]*`) + +// build generates data from source files. +func (w *walker) build(srcs []*source) (*models.PkgInfo, error) { + // Set created time. + w.pinfo.Created = time.Now().UTC() + + // Add source files to walker, I skipped references here. + w.srcs = make(map[string]*source) + for _, src := range srcs { + w.srcs[src.name] = src + } + + w.fset = token.NewFileSet() + + // Find the package and associated files. + ctxt := build.Context{ + GOOS: runtime.GOOS, + GOARCH: runtime.GOARCH, + CgoEnabled: true, + JoinPath: path.Join, + IsAbsPath: path.IsAbs, + SplitPathList: func(list string) []string { return strings.Split(list, ":") }, + IsDir: func(path string) bool { panic("unexpected") }, + HasSubdir: func(root, dir string) (rel string, ok bool) { panic("unexpected") }, + ReadDir: func(dir string) (fi []os.FileInfo, err error) { return w.readDir(dir) }, + OpenFile: func(path string) (r io.ReadCloser, err error) { return w.openFile(path) }, + Compiler: "gc", + } + + bpkg, err := ctxt.ImportDir(w.pinfo.Path, 0) + // Continue if there are no Go source files; we still want the directory info. + _, nogo := err.(*build.NoGoError) + if err != nil { + if nogo { + err = nil + } else { + fmt.Println(w.pinfo) + return w.pinfo, errors.New("doc.walker.build(): " + err.Error()) + } + } + + // Parse the Go files + + files := make(map[string]*ast.File) + for _, name := range append(bpkg.GoFiles, bpkg.CgoFiles...) { + file, err := parser.ParseFile(w.fset, name, w.srcs[name].data, parser.ParseComments) + if err != nil { + //beego.Error("doc.walker.build():", err) + continue + } + files[name] = file + } + + w.pinfo.Imports = bpkg.Imports + fmt.Println(w.pinfo) + // beego.Info("doc.walker.build(", pdoc.ImportPath, "), Goroutine #", runtime.NumGoroutine()) + return w.pinfo, err +} diff --git a/gpm.go b/gpm.go index a9f855513..0c276a900 100644 --- a/gpm.go +++ b/gpm.go @@ -53,7 +53,7 @@ type Command struct { Long string // Flag is a set of flags specific to this command. - Flags []string + Flags map[string]bool } // Name returns the command's name: the first word in the usage line. @@ -79,9 +79,10 @@ func (c *Command) Runnable() bool { } // Commands lists the available commands and help topics. -// The order here is the order in which they are printed by 'go help'. +// The order here is the order in which they are printed by 'gpm help'. var commands = []*Command{ cmdBuild, + cmdInstall, } var exitStatus = 0 @@ -95,11 +96,11 @@ func setExitStatus(n int) { exitMu.Unlock() } -func loadUsage(lang string) bool { +func loadUsage(lang, appPath string) bool { // Load main usage. - f, err := os.Open(appPath + "i18n/usage_" + lang + ".tpl") + f, err := os.Open(appPath + "i18n/" + lang + "/usage.tpl") if err != nil { - fmt.Println(err) + fmt.Println("Load usage:", err) return false } defer f.Close() @@ -111,9 +112,9 @@ func loadUsage(lang string) bool { // Load command usage. for _, cmd := range commands { - f, err := os.Open(appPath + "i18n/usage_" + cmd.Name() + "_" + lang + ".txt") + f, err := os.Open(appPath + "i18n/" + lang + "/usage_" + cmd.Name() + ".txt") if err != nil { - fmt.Println(err) + fmt.Println("Load usage:", err) return false } defer f.Close() @@ -139,13 +140,13 @@ func main() { appPath = strings.Replace(filepath.Dir(appPath), "\\", "/", -1) + "/" // Load configuration. - if _, err := toml.DecodeFile(appPath+"i18n/gpm.toml", &config); err != nil { + if _, err := toml.DecodeFile(appPath+"conf/gpm.toml", &config); err != nil { fmt.Println(err) return } // Load usage template by language. - if !loadUsage(config.Lang) { + if !loadUsage(config.Lang, appPath) { return } @@ -269,3 +270,23 @@ func exit() { } os.Exit(exitStatus) } + +// executeGoCommand executes go commands. +func executeGoCommand(args []string) { + cmdExec := exec.Command("go", args...) + stdout, err := cmdExec.StdoutPipe() + if err != nil { + fmt.Println(err) + } + stderr, err := cmdExec.StderrPipe() + if err != nil { + fmt.Println(err) + } + err = cmdExec.Start() + if err != nil { + fmt.Println(err) + } + go io.Copy(os.Stdout, stdout) + go io.Copy(os.Stderr, stderr) + cmdExec.Wait() +} diff --git a/i18n/usage_en-US.tpl b/i18n/en-US/usage.tpl similarity index 100% rename from i18n/usage_en-US.tpl rename to i18n/en-US/usage.tpl diff --git a/i18n/usage_build_en-US.txt b/i18n/en-US/usage_build.txt similarity index 100% rename from i18n/usage_build_en-US.txt rename to i18n/en-US/usage_build.txt diff --git a/i18n/en-US/usage_install.txt b/i18n/en-US/usage_install.txt new file mode 100644 index 000000000..d3a80ebac --- /dev/null +++ b/i18n/en-US/usage_install.txt @@ -0,0 +1,23 @@ +download and install packages and dependencies||| +Install downloads and installs the packages named by the import paths, +along with their dependencies. + +This command works even you haven't installed any version control tool +such as git, hg, etc. + +The install flags are: + + -p + pure download packages without version control. + -d + download without installing packages. + -u + force to update pakcages. + +The list flags accept a space-separated list of strings. To embed spaces +in an element in the list, surround it with either single or double quotes. + +For more about specifying packages, see 'go help packages'. +For more about hash, see 'gpm help hash'. + +See also: gpm build. diff --git a/install.go b/install.go new file mode 100644 index 000000000..6a5060a8a --- /dev/null +++ b/install.go @@ -0,0 +1,169 @@ +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "net/http" + "os/exec" + "regexp" + "strings" + + "github.com/GPMGo/gpm/doc" + "github.com/GPMGo/gpm/models" + "github.com/GPMGo/gpm/utils" +) + +var ( + isHasGit, isHasHg bool +) + +var cmdInstall = &Command{ + UsageLine: "install [install flags] ", +} + +func init() { + cmdInstall.Run = runInstall + cmdInstall.Flags = map[string]bool{ + "-p": false, + "-d": false, + "-u": false, + } +} + +func runInstall(cmd *Command, args []string) { + // Check if has flags. + num := 0 + for i, f := range args { + if strings.Index(f, "-") > -1 { + // Deal with flags. + if _, ok := cmdInstall.Flags[f]; ok { + cmdInstall.Flags[f] = true + printPrompt(f) + } else { + fmt.Printf("Unknown flag: %s.\n", f) + return + } + num = i + 1 + } + } + // Cut out flag. + args = args[num:] + + // Check length of arguments. + if len(args) < 1 { + fmt.Printf("Please list at least one package.\n") + return + } + + // Check version control tools. + _, err := exec.LookPath("git") + if err == nil { + isHasGit = true + } + _, err = exec.LookPath("hg") + if err == nil { + isHasHg = true + } + + // Install package(s). + for _, p := range args { + // Check if it is a hash string. + // TODO + + // Check if it is vaild remote path. + if !utils.IsValidRemotePath(p) { + fmt.Printf("Invalid remote path: %s.\n", p) + } else { + downloadPackage(p, "") + } + } +} + +func printPrompt(flag string) { + switch flag { + case "-p": + fmt.Println("You enabled pure download.") + case "-d": + fmt.Println("You enabled download without installing.") + } +} + +// downloadPackage download package either use version control tools or not. +func downloadPackage(path, commit string) { + // Check if use version control tools. + switch { + case !cmdInstall.Flags["-p"] && + ((path[0] == 'g' && isHasGit) || (path[0] == 'c' && isHasHg)): // github.com, code.google.com + args := checkGoGetFlags() + args = append(args, path) + fmt.Printf("Installing package: %s.\n", path) + executeGoCommand(args) + default: // Pure download. + if !cmdInstall.Flags["-p"] { + fmt.Printf("No version control tool available, pure download enabled!\n") + } + + fmt.Printf("Downloading package: %s.\n", path) + _, err := pureDownload(path, commit) + if err != nil { + fmt.Printf("Fail to download package(%s) with error: %s.\n", path, err) + } else { + fmt.Printf("Installing package: %s.\n", path) + } + } +} + +func checkGoGetFlags() (args []string) { + args = append(args, "get") + switch { + case cmdInstall.Flags["-d"]: + args = append(args, "-d") + fallthrough + case cmdInstall.Flags["-u"]: + args = append(args, "-u") + } + + return args +} + +// service represents a source code control service. +type service struct { + pattern *regexp.Regexp + prefix string + get func(*http.Client, map[string]string, string) (*models.PkgInfo, error) +} + +// services is the list of source code control services handled by gopkgdoc. +var services = []*service{ + {doc.GithubPattern, "github.com/", doc.GetGithubDoc}, + //{googlePattern, "code.google.com/", getGoogleDoc}, + //{bitbucketPattern, "bitbucket.org/", getBitbucketDoc}, + //{launchpadPattern, "launchpad.net/", getLaunchpadDoc}, +} + +// pureDownload downloads package without control control. +func pureDownload(path, commit string) (pinfo *models.PkgInfo, err error) { + for _, s := range services { + if s.get == nil || !strings.HasPrefix(path, s.prefix) { + continue + } + m := s.pattern.FindStringSubmatch(path) + if m == nil { + if s.prefix != "" { + return nil, doc.NotFoundError{"Import path prefix matches known service, but regexp does not."} + } + continue + } + match := map[string]string{"importPath": path} + for i, n := range s.pattern.SubexpNames() { + if n != "" { + match[n] = m[i] + } + } + return s.get(doc.HttpClient, match, commit) + } + return nil, doc.ErrNoMatch +} diff --git a/models/models.go b/models/models.go new file mode 100644 index 000000000..ec709833c --- /dev/null +++ b/models/models.go @@ -0,0 +1,184 @@ +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package models implemented database access funtions. + +package models + +import ( + "database/sql" + "errors" + //"os" + "strconv" + "strings" + "time" + + "github.com/coocood/qbs" + _ "github.com/mattn/go-sqlite3" +) + +const ( + DB_NAME = "./data/gowalker.db" + _SQLITE3_DRIVER = "sqlite3" +) + +// PkgInfo is package information. +type PkgInfo struct { + Id int64 + Path string `qbs:"index"` // Import path of package. + Imports []string + Note string + Created time.Time `qbs:"index"` // Time when information last updated. + Commit string // Revision tag and project tags. +} + +func connDb() *qbs.Qbs { + // 'sql.Open' only returns error when unknown driver, so it's not necessary to check in other places. + db, err := sql.Open(_SQLITE3_DRIVER, DB_NAME) + if err != nil { + //beego.Error("models.connDb():", err) + } + q := qbs.New(db, qbs.NewSqlite3()) + return q +} + +func setMg() (*qbs.Migration, error) { + db, err := sql.Open(_SQLITE3_DRIVER, DB_NAME) + mg := qbs.NewMigration(db, DB_NAME, qbs.NewSqlite3()) + return mg, err +} + +/*func init() { + // Initialize database. + os.Mkdir("./data", os.ModePerm) + + // Connect to database. + q := connDb() + defer q.Db.Close() + + mg, err := setMg() + if err != nil { + beego.Error("models.init():", err) + } + defer mg.Db.Close() + + // Create data tables. + mg.CreateTableIfNotExists(new(PkgInfo)) + + beego.Trace("Initialized database ->", DB_NAME) +}*/ + +// GetProInfo returns package information from database. +func GetPkgInfo(path string) (*PkgInfo, error) { + // Check path length to reduce connect times. + if len(path) == 0 { + return nil, errors.New("models.GetPkgInfo(): Empty path as not found.") + } + + // Connect to database. + q := connDb() + defer q.Db.Close() + + pinfo := new(PkgInfo) + err := q.WhereEqual("path", path).Find(pinfo) + + return pinfo, err +} + +// GetGroupPkgInfo returns group of package infomration in order to reduce database connect times. +func GetGroupPkgInfo(paths []string) ([]*PkgInfo, error) { + // Connect to database. + q := connDb() + defer q.Db.Close() + + pinfos := make([]*PkgInfo, 0, len(paths)) + for _, v := range paths { + if len(v) > 0 { + pinfo := new(PkgInfo) + err := q.WhereEqual("path", v).Find(pinfo) + if err == nil { + pinfos = append(pinfos, pinfo) + } else { + pinfos = append(pinfos, &PkgInfo{Path: v}) + } + } + } + return pinfos, nil +} + +// GetPkgInfoById returns package information from database by pid. +func GetPkgInfoById(pid int) (*PkgInfo, error) { + // Connect to database. + q := connDb() + defer q.Db.Close() + + pinfo := new(PkgInfo) + err := q.WhereEqual("id", pid).Find(pinfo) + + return pinfo, err +} + +// GetGroupPkgInfoById returns group of package infomration by pid in order to reduce database connect times. +// The formatted pid looks like '$|', so we need to cut '$' here. +func GetGroupPkgInfoById(pids []string) ([]*PkgInfo, error) { + // Connect to database. + q := connDb() + defer q.Db.Close() + + pinfos := make([]*PkgInfo, 0, len(pids)) + for _, v := range pids { + if len(v) > 1 { + pid, err := strconv.Atoi(v[1:]) + if err == nil { + pinfo := new(PkgInfo) + err = q.WhereEqual("id", pid).Find(pinfo) + if err == nil { + pinfos = append(pinfos, pinfo) + } + } + } + } + return pinfos, nil +} + +// DeleteProject deletes everything about the path in database, and update import information. +func DeleteProject(path string) error { + // Check path length to reduce connect times. (except launchpad.net) + if path[0] != 'l' && len(strings.Split(path, "/")) <= 2 { + return errors.New("models.DeleteProject(): Short path as not needed.") + } + + // Connect to database. + q := connDb() + defer q.Db.Close() + + var i1 int64 + // Delete package information. + info := new(PkgInfo) + err := q.WhereEqual("path", path).Find(info) + if err == nil { + i1, err = q.Delete(info) + if err != nil { + //beego.Error("models.DeleteProject(): Information:", err) + } + } + + if i1 > 0 { + //beego.Info("models.DeleteProject(", path, i1, ")") + } + + return nil +} + +// SearchDoc returns packages information that contain keyword +func SearchDoc(key string) ([]*PkgInfo, error) { + // Connect to database. + q := connDb() + defer q.Db.Close() + + var pkgInfos []*PkgInfo + condition := qbs.NewCondition("path like ?", "%"+key+"%") + err := q.Condition(condition).OrderBy("path").FindAll(&pkgInfos) + return pkgInfos, err +} diff --git a/utils/utils.go b/utils/utils.go index 560cc2516..be7056e9e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -6,6 +6,10 @@ package utils import ( "os" + "path" + "regexp" + "runtime" + "strings" ) // IsExist returns if a file or directory exists @@ -13,3 +17,367 @@ func IsExist(path string) bool { _, err := os.Stat(path) return err == nil || os.IsExist(err) } + +var validTLD = map[string]bool{ + // curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt | sed -e '/#/ d' -e 's/.*/"&": true,/' | tr [:upper:] [:lower:] + ".ac": true, + ".ad": true, + ".ae": true, + ".aero": true, + ".af": true, + ".ag": true, + ".ai": true, + ".al": true, + ".am": true, + ".an": true, + ".ao": true, + ".aq": true, + ".ar": true, + ".arpa": true, + ".as": true, + ".asia": true, + ".at": true, + ".au": true, + ".aw": true, + ".ax": true, + ".az": true, + ".ba": true, + ".bb": true, + ".bd": true, + ".be": true, + ".bf": true, + ".bg": true, + ".bh": true, + ".bi": true, + ".biz": true, + ".bj": true, + ".bm": true, + ".bn": true, + ".bo": true, + ".br": true, + ".bs": true, + ".bt": true, + ".bv": true, + ".bw": true, + ".by": true, + ".bz": true, + ".ca": true, + ".cat": true, + ".cc": true, + ".cd": true, + ".cf": true, + ".cg": true, + ".ch": true, + ".ci": true, + ".ck": true, + ".cl": true, + ".cm": true, + ".cn": true, + ".co": true, + ".com": true, + ".coop": true, + ".cr": true, + ".cu": true, + ".cv": true, + ".cw": true, + ".cx": true, + ".cy": true, + ".cz": true, + ".de": true, + ".dj": true, + ".dk": true, + ".dm": true, + ".do": true, + ".dz": true, + ".ec": true, + ".edu": true, + ".ee": true, + ".eg": true, + ".er": true, + ".es": true, + ".et": true, + ".eu": true, + ".fi": true, + ".fj": true, + ".fk": true, + ".fm": true, + ".fo": true, + ".fr": true, + ".ga": true, + ".gb": true, + ".gd": true, + ".ge": true, + ".gf": true, + ".gg": true, + ".gh": true, + ".gi": true, + ".gl": true, + ".gm": true, + ".gn": true, + ".gov": true, + ".gp": true, + ".gq": true, + ".gr": true, + ".gs": true, + ".gt": true, + ".gu": true, + ".gw": true, + ".gy": true, + ".hk": true, + ".hm": true, + ".hn": true, + ".hr": true, + ".ht": true, + ".hu": true, + ".id": true, + ".ie": true, + ".il": true, + ".im": true, + ".in": true, + ".info": true, + ".int": true, + ".io": true, + ".iq": true, + ".ir": true, + ".is": true, + ".it": true, + ".je": true, + ".jm": true, + ".jo": true, + ".jobs": true, + ".jp": true, + ".ke": true, + ".kg": true, + ".kh": true, + ".ki": true, + ".km": true, + ".kn": true, + ".kp": true, + ".kr": true, + ".kw": true, + ".ky": true, + ".kz": true, + ".la": true, + ".lb": true, + ".lc": true, + ".li": true, + ".lk": true, + ".lr": true, + ".ls": true, + ".lt": true, + ".lu": true, + ".lv": true, + ".ly": true, + ".ma": true, + ".mc": true, + ".md": true, + ".me": true, + ".mg": true, + ".mh": true, + ".mil": true, + ".mk": true, + ".ml": true, + ".mm": true, + ".mn": true, + ".mo": true, + ".mobi": true, + ".mp": true, + ".mq": true, + ".mr": true, + ".ms": true, + ".mt": true, + ".mu": true, + ".museum": true, + ".mv": true, + ".mw": true, + ".mx": true, + ".my": true, + ".mz": true, + ".na": true, + ".name": true, + ".nc": true, + ".ne": true, + ".net": true, + ".nf": true, + ".ng": true, + ".ni": true, + ".nl": true, + ".no": true, + ".np": true, + ".nr": true, + ".nu": true, + ".nz": true, + ".om": true, + ".org": true, + ".pa": true, + ".pe": true, + ".pf": true, + ".pg": true, + ".ph": true, + ".pk": true, + ".pl": true, + ".pm": true, + ".pn": true, + ".post": true, + ".pr": true, + ".pro": true, + ".ps": true, + ".pt": true, + ".pw": true, + ".py": true, + ".qa": true, + ".re": true, + ".ro": true, + ".rs": true, + ".ru": true, + ".rw": true, + ".sa": true, + ".sb": true, + ".sc": true, + ".sd": true, + ".se": true, + ".sg": true, + ".sh": true, + ".si": true, + ".sj": true, + ".sk": true, + ".sl": true, + ".sm": true, + ".sn": true, + ".so": true, + ".sr": true, + ".st": true, + ".su": true, + ".sv": true, + ".sx": true, + ".sy": true, + ".sz": true, + ".tc": true, + ".td": true, + ".tel": true, + ".tf": true, + ".tg": true, + ".th": true, + ".tj": true, + ".tk": true, + ".tl": true, + ".tm": true, + ".tn": true, + ".to": true, + ".tp": true, + ".tr": true, + ".travel": true, + ".tt": true, + ".tv": true, + ".tw": true, + ".tz": true, + ".ua": true, + ".ug": true, + ".uk": true, + ".us": true, + ".uy": true, + ".uz": true, + ".va": true, + ".vc": true, + ".ve": true, + ".vg": true, + ".vi": true, + ".vn": true, + ".vu": true, + ".wf": true, + ".ws": true, + ".xn--0zwm56d": true, + ".xn--11b5bs3a9aj6g": true, + ".xn--3e0b707e": true, + ".xn--45brj9c": true, + ".xn--80akhbyknj4f": true, + ".xn--80ao21a": true, + ".xn--90a3ac": true, + ".xn--9t4b11yi5a": true, + ".xn--clchc0ea0b2g2a9gcd": true, + ".xn--deba0ad": true, + ".xn--fiqs8s": true, + ".xn--fiqz9s": true, + ".xn--fpcrj9c3d": true, + ".xn--fzc2c9e2c": true, + ".xn--g6w251d": true, + ".xn--gecrj9c": true, + ".xn--h2brj9c": true, + ".xn--hgbk6aj7f53bba": true, + ".xn--hlcj6aya9esc7a": true, + ".xn--j6w193g": true, + ".xn--jxalpdlp": true, + ".xn--kgbechtv": true, + ".xn--kprw13d": true, + ".xn--kpry57d": true, + ".xn--lgbbat1ad8j": true, + ".xn--mgb9awbf": true, + ".xn--mgbaam7a8h": true, + ".xn--mgbayh7gpa": true, + ".xn--mgbbh1a71e": true, + ".xn--mgbc0a9azcg": true, + ".xn--mgberp4a5d4ar": true, + ".xn--mgbx4cd0ab": true, + ".xn--o3cw4h": true, + ".xn--ogbpf8fl": true, + ".xn--p1ai": true, + ".xn--pgbs0dh": true, + ".xn--s9brj9c": true, + ".xn--wgbh1c": true, + ".xn--wgbl6a": true, + ".xn--xkc2al3hye2a": true, + ".xn--xkc2dl3a5ee0h": true, + ".xn--yfro4i67o": true, + ".xn--ygbi2ammx": true, + ".xn--zckzah": true, + ".xxx": true, + ".ye": true, + ".yt": true, + ".za": true, + ".zm": true, + ".zw": true, +} + +var ( + validHost = regexp.MustCompile(`^[-a-z0-9]+(?:\.[-a-z0-9]+)+$`) + validPathElement = regexp.MustCompile(`^[-A-Za-z0-9~+][-A-Za-z0-9_.]*$`) +) + +// IsValidRemotePath returns true if importPath is structurally valid for "go get". +func IsValidRemotePath(importPath string) bool { + + parts := strings.Split(importPath, "/") + + if len(parts) <= 1 { + // Import path must contain at least one "/". + return false + } + + if !validTLD[path.Ext(parts[0])] { + return false + } + + if !validHost.MatchString(parts[0]) { + return false + } + for _, part := range parts[1:] { + if !validPathElement.MatchString(part) || part == "testdata" { + return false + } + } + + return true +} + +// GetGOPATHs return all GOPATH in system. +func GetGOPATH() []string { + gopath := os.Getenv("GOPATH") + var paths []string + if runtime.GOOS == "windows" { + gopath = strings.Replace(gopath, "\\", "/", -1) + paths = strings.Split(gopath, ";") + } else { + paths = strings.Split(gopath, ":") + } + return paths +} diff --git a/utils/utils_test.go b/utils/utils_test.go new file mode 100644 index 000000000..d2719c4c9 --- /dev/null +++ b/utils/utils_test.go @@ -0,0 +1,24 @@ +// Copyright (c) 2013 GPMGo Members. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package utils + +import ( + "testing" +) + +var RemotePaths = []string{ + "github.com/coocood/qbs", + "code.google.com/p/draw2d", + "launchpad.net/goamz", + "bitbucket.org/gotamer/conv", +} + +func TestIsValidRemotePath(t *testing.T) { + for _, p := range RemotePaths { + if !IsValidRemotePath(p) { + t.Errorf("Invalid remote path: %s", p) + } + } +}