Fix #362, update code with upstream

modules/base/markdown.go

@@ -39,7 +39,7 @@ func isLink(link []byte) bool {
 func IsMarkdownFile(name string) bool {
 	name = strings.ToLower(name)
 	switch filepath.Ext(name) {
-	case ".md", ".markdown", ".mdown":
+	case ".md", ".markdown", ".mdown", ".mkd":
 		return true
 	}
 	return false

modules/httplib/httplib.go

@@ -5,6 +5,8 @@
 
 package httplib
 
+// NOTE: last sync c07b1d8 on Aug 23, 2014.
+
 import (
 	"bytes"
 	"crypto/tls"
@@ -12,60 +14,90 @@ import (
 	"encoding/xml"
 	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net"
 	"net/http"
+	"net/http/cookiejar"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"time"
 )
 
-var defaultUserAgent = "gogsServer"
+var defaultSetting = BeegoHttpSettings{false, "beegoServer", 60 * time.Second, 60 * time.Second, nil, nil, nil, false}
+var defaultCookieJar http.CookieJar
+var settingMutex sync.Mutex
+
+// createDefaultCookie creates a global cookiejar to store cookies.
+func createDefaultCookie() {
+	settingMutex.Lock()
+	defer settingMutex.Unlock()
+	defaultCookieJar, _ = cookiejar.New(nil)
+}
+
+// Overwrite default settings
+func SetDefaultSetting(setting BeegoHttpSettings) {
+	settingMutex.Lock()
+	defer settingMutex.Unlock()
+	defaultSetting = setting
+	if defaultSetting.ConnectTimeout == 0 {
+		defaultSetting.ConnectTimeout = 60 * time.Second
+	}
+	if defaultSetting.ReadWriteTimeout == 0 {
+		defaultSetting.ReadWriteTimeout = 60 * time.Second
+	}
+}
+
+// return *BeegoHttpRequest with specific method
+func newBeegoRequest(url, method string) *BeegoHttpRequest {
+	var resp http.Response
+	req := http.Request{
+		Method:     method,
+		Header:     make(http.Header),
+		Proto:      "HTTP/1.1",
+		ProtoMajor: 1,
+		ProtoMinor: 1,
+	}
+	return &BeegoHttpRequest{url, &req, map[string]string{}, map[string]string{}, defaultSetting, &resp, nil}
+}
 
 // Get returns *BeegoHttpRequest with GET method.
 func Get(url string) *BeegoHttpRequest {
-	var req http.Request
+	return newBeegoRequest(url, "GET")
-	req.Method = "GET"
-	req.Header = http.Header{}
-	req.Header.Set("User-Agent", defaultUserAgent)
-	return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
 }
 
 // Post returns *BeegoHttpRequest with POST method.
 func Post(url string) *BeegoHttpRequest {
-	var req http.Request
+	return newBeegoRequest(url, "POST")
-	req.Method = "POST"
-	req.Header = http.Header{}
-	req.Header.Set("User-Agent", defaultUserAgent)
-	return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
 }
 
 // Put returns *BeegoHttpRequest with PUT method.
 func Put(url string) *BeegoHttpRequest {
-	var req http.Request
+	return newBeegoRequest(url, "PUT")
-	req.Method = "PUT"
-	req.Header = http.Header{}
-	req.Header.Set("User-Agent", defaultUserAgent)
-	return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
 }
 
-// Delete returns *BeegoHttpRequest DELETE GET method.
+// Delete returns *BeegoHttpRequest DELETE method.
 func Delete(url string) *BeegoHttpRequest {
-	var req http.Request
+	return newBeegoRequest(url, "DELETE")
-	req.Method = "DELETE"
-	req.Header = http.Header{}
-	req.Header.Set("User-Agent", defaultUserAgent)
-	return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
 }
 
 // Head returns *BeegoHttpRequest with HEAD method.
 func Head(url string) *BeegoHttpRequest {
-	var req http.Request
+	return newBeegoRequest(url, "HEAD")
-	req.Method = "HEAD"
+}
-	req.Header = http.Header{}
+
-	req.Header.Set("User-Agent", defaultUserAgent)
+// BeegoHttpSettings
-	return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+type BeegoHttpSettings struct {
+	ShowDebug        bool
+	UserAgent        string
+	ConnectTimeout   time.Duration
+	ReadWriteTimeout time.Duration
+	TlsClientConfig  *tls.Config
+	Proxy            func(*http.Request) (*url.URL, error)
+	Transport        http.RoundTripper
+	EnableCookie     bool
 }
 
 // BeegoHttpRequest provides more useful methods for requesting one url than http.Request.
@@ -73,30 +105,52 @@ type BeegoHttpRequest struct {
 	url     string
 	req     *http.Request
 	params  map[string]string
-	showdebug        bool
+	files   map[string]string
-	connectTimeout   time.Duration
+	setting BeegoHttpSettings
-	readWriteTimeout time.Duration
+	resp    *http.Response
-	tlsClientConfig  *tls.Config
+	body    []byte
-	proxy            func(*http.Request) (*url.URL, error)
+}
-	transport        http.RoundTripper
+
+// Change request settings
+func (b *BeegoHttpRequest) Setting(setting BeegoHttpSettings) *BeegoHttpRequest {
+	b.setting = setting
+	return b
+}
+
+// SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password.
+func (b *BeegoHttpRequest) SetBasicAuth(username, password string) *BeegoHttpRequest {
+	b.req.SetBasicAuth(username, password)
+	return b
+}
+
+// SetEnableCookie sets enable/disable cookiejar
+func (b *BeegoHttpRequest) SetEnableCookie(enable bool) *BeegoHttpRequest {
+	b.setting.EnableCookie = enable
+	return b
+}
+
+// SetUserAgent sets User-Agent header field
+func (b *BeegoHttpRequest) SetUserAgent(useragent string) *BeegoHttpRequest {
+	b.setting.UserAgent = useragent
+	return b
 }
 
 // Debug sets show debug or not when executing request.
 func (b *BeegoHttpRequest) Debug(isdebug bool) *BeegoHttpRequest {
-	b.showdebug = isdebug
+	b.setting.ShowDebug = isdebug
 	return b
 }
 
 // SetTimeout sets connect time out and read-write time out for BeegoRequest.
 func (b *BeegoHttpRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHttpRequest {
-	b.connectTimeout = connectTimeout
+	b.setting.ConnectTimeout = connectTimeout
-	b.readWriteTimeout = readWriteTimeout
+	b.setting.ReadWriteTimeout = readWriteTimeout
 	return b
 }
 
 // SetTLSClientConfig sets tls connection configurations if visiting https url.
 func (b *BeegoHttpRequest) SetTLSClientConfig(config *tls.Config) *BeegoHttpRequest {
-	b.tlsClientConfig = config
+	b.setting.TlsClientConfig = config
 	return b
 }
 
@@ -106,6 +160,23 @@ func (b *BeegoHttpRequest) Header(key, value string) *BeegoHttpRequest {
 	return b
 }
 
+// Set the protocol version for incoming requests.
+// Client requests always use HTTP/1.1.
+func (b *BeegoHttpRequest) SetProtocolVersion(vers string) *BeegoHttpRequest {
+	if len(vers) == 0 {
+		vers = "HTTP/1.1"
+	}
+
+	major, minor, ok := http.ParseHTTPVersion(vers)
+	if ok {
+		b.req.Proto = vers
+		b.req.ProtoMajor = major
+		b.req.ProtoMinor = minor
+	}
+
+	return b
+}
+
 // SetCookie add cookie into request.
 func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest {
 	b.req.Header.Add("Cookie", cookie.String())
@@ -114,7 +185,7 @@ func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest {
 
 // Set transport to
 func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpRequest {
-	b.transport = transport
+	b.setting.Transport = transport
 	return b
 }
 
@@ -126,7 +197,7 @@ func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpR
 // 		return u, nil
 // 	}
 func (b *BeegoHttpRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHttpRequest {
-	b.proxy = proxy
+	b.setting.Proxy = proxy
 	return b
 }
 
@@ -137,6 +208,11 @@ func (b *BeegoHttpRequest) Param(key, value string) *BeegoHttpRequest {
 	return b
 }
 
+func (b *BeegoHttpRequest) PostFile(formname, filename string) *BeegoHttpRequest {
+	b.files[formname] = filename
+	return b
+}
+
 // Body adds request raw body.
 // it supports string and []byte.
 func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest {
@@ -154,6 +230,9 @@ func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest {
 }
 
 func (b *BeegoHttpRequest) getResponse() (*http.Response, error) {
+	if b.resp.StatusCode != 0 {
+		return b.resp, nil
+	}
 	var paramBody string
 	if len(b.params) > 0 {
 		var buf bytes.Buffer
@@ -174,60 +253,102 @@ func (b *BeegoHttpRequest) getResponse() (*http.Response, error) {
 			b.url = b.url + "?" + paramBody
 		}
 	} else if b.req.Method == "POST" && b.req.Body == nil && len(paramBody) > 0 {
+		if len(b.files) > 0 {
+			bodyBuf := &bytes.Buffer{}
+			bodyWriter := multipart.NewWriter(bodyBuf)
+			for formname, filename := range b.files {
+				fileWriter, err := bodyWriter.CreateFormFile(formname, filename)
+				if err != nil {
+					return nil, err
+				}
+				fh, err := os.Open(filename)
+				if err != nil {
+					return nil, err
+				}
+				//iocopy
+				_, err = io.Copy(fileWriter, fh)
+				fh.Close()
+				if err != nil {
+					return nil, err
+				}
+			}
+			for k, v := range b.params {
+				bodyWriter.WriteField(k, v)
+			}
+			contentType := bodyWriter.FormDataContentType()
+			bodyWriter.Close()
+			b.Header("Content-Type", contentType)
+			b.req.Body = ioutil.NopCloser(bodyBuf)
+			b.req.ContentLength = int64(bodyBuf.Len())
+		} else {
 			b.Header("Content-Type", "application/x-www-form-urlencoded")
 			b.Body(paramBody)
 		}
+	}
 
 	url, err := url.Parse(b.url)
-	if url.Scheme == "" {
-		b.url = "http://" + b.url
-		url, err = url.Parse(b.url)
-	}
 	if err != nil {
 		return nil, err
 	}
 
 	b.req.URL = url
-	if b.showdebug {
-		dump, err := httputil.DumpRequest(b.req, true)
-		if err != nil {
-			println(err.Error())
-		}
-		println(string(dump))
-	}
 
-	trans := b.transport
+	trans := b.setting.Transport
 
 	if trans == nil {
 		// create default transport
 		trans = &http.Transport{
-			TLSClientConfig: b.tlsClientConfig,
+			TLSClientConfig: b.setting.TlsClientConfig,
-			Proxy:           b.proxy,
+			Proxy:           b.setting.Proxy,
-			Dial:            TimeoutDialer(b.connectTimeout, b.readWriteTimeout),
+			Dial:            TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
 		}
 	} else {
 		// if b.transport is *http.Transport then set the settings.
 		if t, ok := trans.(*http.Transport); ok {
 			if t.TLSClientConfig == nil {
-				t.TLSClientConfig = b.tlsClientConfig
+				t.TLSClientConfig = b.setting.TlsClientConfig
 			}
 			if t.Proxy == nil {
-				t.Proxy = b.proxy
+				t.Proxy = b.setting.Proxy
 			}
 			if t.Dial == nil {
-				t.Dial = TimeoutDialer(b.connectTimeout, b.readWriteTimeout)
+				t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout)
+			}
 		}
 	}
+
+	var jar http.CookieJar
+	if b.setting.EnableCookie {
+		if defaultCookieJar == nil {
+			createDefaultCookie()
+		}
+		jar = defaultCookieJar
+	} else {
+		jar = nil
 	}
 
 	client := &http.Client{
 		Transport: trans,
+		Jar:       jar,
+	}
+
+	if b.setting.UserAgent != "" {
+		b.req.Header.Set("User-Agent", b.setting.UserAgent)
+	}
+
+	if b.setting.ShowDebug {
+		dump, err := httputil.DumpRequest(b.req, true)
+		if err != nil {
+			println(err.Error())
+		}
+		println(string(dump))
 	}
 
 	resp, err := client.Do(b.req)
 	if err != nil {
 		return nil, err
 	}
+	b.resp = resp
 	return resp, nil
 }
 
@@ -245,6 +366,9 @@ func (b *BeegoHttpRequest) String() (string, error) {
 // Bytes returns the body []byte in response.
 // it calls Response inner.
 func (b *BeegoHttpRequest) Bytes() ([]byte, error) {
+	if b.body != nil {
+		return b.body, nil
+	}
 	resp, err := b.getResponse()
 	if err != nil {
 		return nil, err
@@ -257,6 +381,7 @@ func (b *BeegoHttpRequest) Bytes() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	b.body = data
 	return data, nil
 }
 
@@ -278,11 +403,8 @@ func (b *BeegoHttpRequest) ToFile(filename string) error {
 	}
 	defer resp.Body.Close()
 	_, err = io.Copy(f, resp.Body)
-	if err != nil {
 	return err
 }
-	return nil
-}
 
 // ToJson returns the map that marshals from the body bytes as json in response .
 // it calls Response inner.
@@ -292,25 +414,19 @@ func (b *BeegoHttpRequest) ToJson(v interface{}) error {
 		return err
 	}
 	err = json.Unmarshal(data, v)
-	if err != nil {
 	return err
 }
-	return nil
-}
 
 // ToXml returns the map that marshals from the body bytes as xml in response .
 // it calls Response inner.
-func (b *BeegoHttpRequest) ToXML(v interface{}) error {
+func (b *BeegoHttpRequest) ToXml(v interface{}) error {
 	data, err := b.Bytes()
 	if err != nil {
 		return err
 	}
 	err = xml.Unmarshal(data, v)
-	if err != nil {
 	return err
 }
-	return nil
-}
 
 // Response executes request client gets response mannually.
 func (b *BeegoHttpRequest) Response() (*http.Response, error) {

modules/httplib/httplib_test.go

@@ -1,32 +1,196 @@
+// Copyright 2013 The Beego Authors. All rights reserved.
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
 package httplib
 
 import (
 	"io/ioutil"
+	"os"
+	"strings"
 	"testing"
 )
 
-func TestGetUrl(t *testing.T) {
+func TestResponse(t *testing.T) {
-	resp, err := Get("http://beego.me/").Debug(true).Response()
+	req := Get("http://httpbin.org/get")
+	resp, err := req.Response()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(resp)
+}
+
+func TestGet(t *testing.T) {
+	req := Get("http://httpbin.org/get")
+	b, err := req.Bytes()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(b)
+
+	s, err := req.String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(s)
+
+	if string(b) != s {
+		t.Fatal("request data not match")
+	}
+}
+
+func TestSimplePost(t *testing.T) {
+	v := "smallfish"
+	req := Post("http://httpbin.org/post")
+	req.Param("username", v)
+
+	str, err := req.String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(str)
+
+	n := strings.Index(str, v)
+	if n == -1 {
+		t.Fatal(v + " not found in post")
+	}
+}
+
+func TestPostFile(t *testing.T) {
+	v := "smallfish"
+	req := Post("http://httpbin.org/post")
+	req.Param("username", v)
+	req.PostFile("uploadfile", "httplib_test.go")
+
+	str, err := req.String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(str)
+
+	n := strings.Index(str, v)
+	if n == -1 {
+		t.Fatal(v + " not found in post")
+	}
+}
+
+func TestSimplePut(t *testing.T) {
+	str, err := Put("http://httpbin.org/put").String()
 	if err != nil {
 		t.Fatal(err)
 	}
-	if resp.Body == nil {
+	t.Log(str)
-		t.Fatal("body is nil")
 }
-	data, err := ioutil.ReadAll(resp.Body)
+
-	defer resp.Body.Close()
+func TestSimpleDelete(t *testing.T) {
+	str, err := Delete("http://httpbin.org/delete").String()
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(data) == 0 {
+	t.Log(str)
-		t.Fatal("data is no")
+}
+
+func TestWithCookie(t *testing.T) {
+	v := "smallfish"
+	str, err := Get("http://httpbin.org/cookies/set?k1=" + v).SetEnableCookie(true).String()
+	if err != nil {
+		t.Fatal(err)
 	}
+	t.Log(str)
 
-	str, err := Get("http://beego.me/").String()
+	str, err = Get("http://httpbin.org/cookies").SetEnableCookie(true).String()
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(str) == 0 {
+	t.Log(str)
-		t.Fatal("has no info")
+
+	n := strings.Index(str, v)
+	if n == -1 {
+		t.Fatal(v + " not found in cookie")
+	}
+}
+
+func TestWithBasicAuth(t *testing.T) {
+	str, err := Get("http://httpbin.org/basic-auth/user/passwd").SetBasicAuth("user", "passwd").String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(str)
+	n := strings.Index(str, "authenticated")
+	if n == -1 {
+		t.Fatal("authenticated not found in response")
+	}
+}
+
+func TestWithUserAgent(t *testing.T) {
+	v := "beego"
+	str, err := Get("http://httpbin.org/headers").SetUserAgent(v).String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(str)
+
+	n := strings.Index(str, v)
+	if n == -1 {
+		t.Fatal(v + " not found in user-agent")
+	}
+}
+
+func TestWithSetting(t *testing.T) {
+	v := "beego"
+	var setting BeegoHttpSettings
+	setting.EnableCookie = true
+	setting.UserAgent = v
+	setting.Transport = nil
+	SetDefaultSetting(setting)
+
+	str, err := Get("http://httpbin.org/get").String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(str)
+
+	n := strings.Index(str, v)
+	if n == -1 {
+		t.Fatal(v + " not found in user-agent")
+	}
+}
+
+func TestToJson(t *testing.T) {
+	req := Get("http://httpbin.org/ip")
+	resp, err := req.Response()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(resp)
+
+	// httpbin will return http remote addr
+	type Ip struct {
+		Origin string `json:"origin"`
+	}
+	var ip Ip
+	err = req.ToJson(&ip)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(ip.Origin)
+
+	if n := strings.Count(ip.Origin, "."); n != 3 {
+		t.Fatal("response is not valid ip")
+	}
+}
+
+func TestToFile(t *testing.T) {
+	f := "beego_testfile"
+	req := Get("http://httpbin.org/ip")
+	err := req.ToFile(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(f)
+	b, err := ioutil.ReadFile(f)
+	if n := strings.Index(string(b), "origin"); n == -1 {
+		t.Fatal(err)
 	}
 }