Added oschina repo download

doc/bitbucket.go

@@ -28,12 +28,12 @@ import (
 )
 
 var (
-	BitbucketPattern = regexp.MustCompile(`^bitbucket\.org/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`)
+	bitbucketPattern = regexp.MustCompile(`^bitbucket\.org/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`)
 	bitbucketEtagRe  = regexp.MustCompile(`^(hg|git)-`)
 )
 
-// GetBitbucketDoc downloads tarball from bitbucket.org.
+// getBitbucketDoc downloads tarball from bitbucket.org.
-func GetBitbucketDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
+func getBitbucketDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
 	// Check version control.
 	if m := bitbucketEtagRe.FindStringSubmatch(nod.Value); m != nil {
 		match["vcs"] = m[1]

doc/github.go

@@ -28,7 +28,7 @@ import (
 
 var (
 	githubRawHeader = http.Header{"Accept": {"application/vnd.github-blob.raw"}}
-	GithubPattern   = regexp.MustCompile(`^github\.com/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`)
+	githubPattern   = regexp.MustCompile(`^github\.com/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`)
 	githubCred      string
 )
 
@@ -42,64 +42,32 @@ func SetGithubCredentials(token string) {
 	}
 }
 
-// GetGithubDoc downloads tarball from github.com.
+// getGithubDoc downloads tarball from github.com.
-func GetGithubDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
+func getGithubDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
 	match["cred"] = githubCred
 
-	if nod.Type == BRANCH {
+	// Check downlaod type.
+	switch nod.Type {
+	case BRANCH:
 		if len(nod.Value) == 0 {
 			match["sha"] = MASTER
 		} else {
 			match["sha"] = nod.Value
 		}
-	}
+	case TAG, COMMIT:
-
+		match["sha"] = nod.Value
-	// JSON struct for github.com.
+	default:
-	var refs []*struct {
+		return nil, errors.New("Unknown node type: " + nod.Type)
-		Ref    string
-		Url    string
-		Object struct {
-			Sha  string
-			Type string
-			Url  string
-		}
-	}
-
-	if nod.IsGetDeps {
-		if nod.Type == COMMIT {
-			// Get up-to-date version.
-			err := httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/git/refs?{cred}", match), &refs)
-			if err != nil {
-				return nil, err
-			}
-
-			for _, ref := range refs {
-				if strings.HasPrefix(ref.Ref, "refs/heads/master") {
-					match["sha"] = ref.Object.Sha
-					break
-				}
-			}
-
-			nod.Value = match["sha"]
-		}
-	} else {
-		// Check downlaod type.
-		switch nod.Type {
-		case TAG, COMMIT, BRANCH:
-			match["sha"] = nod.Value
-		default:
-			return nil, errors.New("Unknown node type: " + nod.Type)
-		}
 	}
 
 	// We use .zip here.
-	// zip : https://github.com/{owner}/{repo}/archive/{sha}.zip
+	// zip: https://github.com/{owner}/{repo}/archive/{sha}.zip
-	// tarball : https://github.com/{owner}/{repo}/tarball/{sha}
+	// tarball: https://github.com/{owner}/{repo}/tarball/{sha}
 
 	// Downlaod archive.
 	p, err := HttpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil)
 	if err != nil {
-		return nil, err
+		return nil, errors.New("Fail to donwload Github repo -> " + err.Error())
 	}
 
 	shaName := expand("{repo}-{sha}", match)

doc/google.go

@@ -28,7 +28,7 @@ var (
 	googleEtagRe     = regexp.MustCompile(`^(hg|git|svn)-`)
 	googleFileRe     = regexp.MustCompile(`<li><a href="([^"/]+)"`)
 	googleDirRe      = regexp.MustCompile(`<li><a href="([^".]+)"`)
-	GooglePattern    = regexp.MustCompile(`^code\.google\.com/p/(?P<repo>[a-z0-9\-]+)(:?\.(?P<subrepo>[a-z0-9\-]+))?(?P<dir>/[a-z0-9A-Z_.\-/]+)?$`)
+	googlePattern    = regexp.MustCompile(`^code\.google\.com/p/(?P<repo>[a-z0-9\-]+)(:?\.(?P<subrepo>[a-z0-9\-]+))?(?P<dir>/[a-z0-9A-Z_.\-/]+)?$`)
 )
 
 func setupGoogleMatch(match map[string]string) {
@@ -56,8 +56,8 @@ func getGoogleVCS(client *http.Client, match map[string]string) error {
 	return nil
 }
 
-// GetGoogleDoc downloads raw files from code.google.com.
+// getGoogleDoc downloads raw files from code.google.com.
-func GetGoogleDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
+func getGoogleDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
 	setupGoogleMatch(match)
 	// Check version control.
 	if m := googleEtagRe.FindStringSubmatch(nod.Value); m != nil {

doc/launchpad.go

@@ -26,10 +26,10 @@ import (
 	"strings"
 )
 
-var LaunchpadPattern = regexp.MustCompile(`^launchpad\.net/(?P<repo>(?P<project>[a-z0-9A-Z_.\-]+)(?P<series>/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]+)*$`)
+var launchpadPattern = regexp.MustCompile(`^launchpad\.net/(?P<repo>(?P<project>[a-z0-9A-Z_.\-]+)(?P<series>/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]+)*$`)
 
-// GetLaunchpadDoc downloads tarball from launchpad.net.
+// getLaunchpadDoc downloads tarball from launchpad.net.
-func GetLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
+func getLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
 
 	if match["project"] != "" && match["series"] != "" {
 		rc, err := httpGet(client, expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil)
@@ -101,8 +101,6 @@ func GetLaunchpadDoc(client *http.Client, match map[string]string, installRepoPa
 				dirs = append(dirs, absPath)
 			}
 		case !strings.HasPrefix(fn, "."):
-			//os.MkdirAll(path.Dir(absPath)+"/", os.ModePerm)
-
 			// Get data from archive.
 			fbytes := make([]byte, h.Size)
 			if _, err := io.ReadFull(tr, fbytes); err != nil {

doc/oschina.go

@@ -0,0 +1,129 @@
+// Copyright 2013 gopm authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package doc
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"io"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+)
+
+var (
+	oscTagRe   = regexp.MustCompile(`/repository/archive\?ref=(.*)">`)
+	oscPattern = regexp.MustCompile(`^git\.oschina\.net/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`)
+)
+
+// getGithubDoc downloads tarball from git.oschina.com.
+func getOSCDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, cmdFlags map[string]bool) ([]string, error) {
+	// Check downlaod type.
+	switch nod.Type {
+	case BRANCH:
+		if len(nod.Value) == 0 {
+			match["sha"] = MASTER
+		} else {
+			match["sha"] = nod.Value
+		}
+	case TAG, COMMIT:
+		match["sha"] = nod.Value
+	default:
+		return nil, errors.New("Unknown node type: " + nod.Type)
+	}
+
+	// zip: http://{projectRoot}/repository/archive?ref={sha}
+
+	// Downlaod archive.
+	p, err := HttpGetBytes(client, expand("http://git.oschina.net/{owner}/{repo}/repository/archive?ref={sha}", match), nil)
+	if err != nil {
+		return nil, errors.New("Fail to donwload OSChina repo -> " + err.Error())
+	}
+
+	suf := "." + nod.Value
+	if len(suf) == 1 {
+		suf = ""
+	}
+
+	projectPath := expand("git.oschina.net/{owner}/{repo}", match)
+	installPath := installRepoPath + "/" + projectPath + suf
+	nod.ImportPath = projectPath
+
+	// Remove old files.
+	os.RemoveAll(installPath + "/")
+	os.MkdirAll(installPath+"/", os.ModePerm)
+
+	r, err := zip.NewReader(bytes.NewReader(p), int64(len(p)))
+	if err != nil {
+		return nil, errors.New("Fail to unzip OSChina repo -> " + err.Error())
+	}
+
+	nameLen := len(match["repo"])
+	dirs := make([]string, 0, 5)
+	// Need to add root path because we cannot get from tarball.
+	dirs = append(dirs, installPath+"/")
+	for _, f := range r.File {
+		fileName := f.FileInfo().Name()[nameLen+1:]
+		absPath := installPath + "/" + fileName
+
+		if strings.HasSuffix(absPath, "/") {
+			dirs = append(dirs, absPath)
+			os.MkdirAll(absPath, os.ModePerm)
+			continue
+		}
+		// d, _ := path.Split(absPath)
+		// if !checkDir(d, dirs) {
+		// 	dirs = append(dirs, d)
+		// 	os.MkdirAll(d, os.ModePerm)
+		// }
+
+		// Get file from archive.
+		rc, err := f.Open()
+		if err != nil {
+			return nil, errors.New("Fail to open OSChina repo -> " + err.Error())
+		}
+
+		// Write data to file
+		fw, _ := os.Create(absPath)
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = io.Copy(fw, rc)
+		// Close files.
+		rc.Close()
+		fw.Close()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var imports []string
+
+	// Check if need to check imports.
+	if nod.IsGetDeps {
+		for _, d := range dirs {
+			importPkgs, err := CheckImports(d, match["importPath"])
+			if err != nil {
+				return nil, err
+			}
+			imports = append(imports, importPkgs...)
+		}
+	}
+
+	return imports, err
+}

doc/struct.go

@@ -69,8 +69,9 @@ type service struct {
 
 // services is the list of source code control services handled by gopkgdoc.
 var services = []*service{
-	{GithubPattern, "github.com/", GetGithubDoc},
+	{githubPattern, "github.com/", getGithubDoc},
-	{GooglePattern, "code.google.com/", GetGoogleDoc},
+	{googlePattern, "code.google.com/", getGoogleDoc},
-	{BitbucketPattern, "bitbucket.org/", GetBitbucketDoc},
+	{bitbucketPattern, "bitbucket.org/", getBitbucketDoc},
-	{LaunchpadPattern, "launchpad.net/", GetLaunchpadDoc},
+	{launchpadPattern, "launchpad.net/", getLaunchpadDoc},
+	{oscPattern, "git.oschina.net/", getOSCDoc},
 }

gopm.go

@@ -37,7 +37,7 @@ import (
 
 // Test that go1.1 tag above is included in builds. main.go refers to this definition.
 const go11tag = true
-const APP_VER = "0.2.0.0818"
+const APP_VER = "0.2.2.0819"
 
 var (
 	config map[string]interface{}