Merge pull request #11413 from grafana/cp-5.0.4

Cherry-picks for v5.0.4

Gopkg.lock

@@ -171,12 +171,14 @@
 [[projects]]
   name = "github.com/go-xorm/core"
   packages = ["."]
-  revision = "e8409d73255791843585964791443dbad877058c"
+  revision = "da1adaf7a28ca792961721a34e6e04945200c890"
+  version = "v0.5.7"
 
 [[projects]]
   name = "github.com/go-xorm/xorm"
   packages = ["."]
-  revision = "6687a2b4e824f4d87f2d65060ec5cb0d896dff1e"
+  revision = "1933dd69e294c0a26c0266637067f24dbb25770c"
+  version = "v0.6.4"
 
 [[projects]]
   branch = "master"
@@ -631,6 +633,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "4de68f1342ba98a637ec8ca7496aeeae2021bf9e4c7c80db7924e14709151a62"
+  inputs-digest = "112ccff73f668c8c4dbe3d41c37ebee65fd7d839f5a4fa0665c593cae0095dad"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -85,13 +85,11 @@ ignored = [
 
 [[constraint]]
   name = "github.com/go-xorm/core"
-  revision = "e8409d73255791843585964791443dbad877058c"
+  version = "0.5.7"
-  #version = "0.5.7" //keeping this since we would rather depend on version then commit
 
 [[constraint]]
   name = "github.com/go-xorm/xorm"
-  revision = "6687a2b4e824f4d87f2d65060ec5cb0d896dff1e"
+  version = "0.6.4"
-  #version = "0.6.4" //keeping this since we would rather depend on version then commit
 
 [[constraint]]
   name = "github.com/gorilla/websocket"

conf/defaults.ini

@@ -82,6 +82,9 @@ max_idle_conn = 2
 # Max conn setting default is 0 (mean not set)
 max_open_conn =
 
+# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
+conn_max_lifetime = 14400
+
 # Set to true to log the sql calls and execution times.
 log_queries =
 
@@ -125,6 +128,9 @@ cookie_secure = false
 session_life_time = 86400
 gc_interval_time = 86400
 
+# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
+conn_max_lifetime = 14400
+
 #################################### Data proxy ###########################
 [dataproxy]
 

conf/sample.ini

@@ -90,6 +90,9 @@
 # Max conn setting default is 0 (mean not set)
 ;max_open_conn =
 
+# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
+;conn_max_lifetime = 14400
+
 # Set to true to log the sql calls and execution times.
 log_queries =
 

docs/sources/installation/configuration.md

@@ -234,7 +234,12 @@ The maximum number of connections in the idle connection pool.
 ### max_open_conn
 The maximum number of open connections to the database.
 
+### conn_max_lifetime
+
+Sets the maximum amount of time a connection may be reused. The default is 14400 (which means 14400 seconds or 4 hours). For MySQL, this setting should be shorter than the [`wait_timeout`](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_wait_timeout) variable.
+
 ### log_queries
+
 Set to `true` to log the sql calls and execution times.
 
 <hr />

package.json

@@ -4,7 +4,7 @@
     "company": "Grafana Labs"
   },
   "name": "grafana",
-  "version": "5.0.0",
+  "version": "5.0.4",
   "repository": {
     "type": "git",
     "url": "http://github.com/grafana/grafana.git"

packaging/publish/publish_both.sh

@@ -1,5 +1,5 @@
 #! /usr/bin/env bash
-version=4.6.3
+version=5.0.0
 
 wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_${version}_amd64.deb
 

pkg/api/api.go

@@ -66,6 +66,7 @@ func (hs *HttpServer) registerRoutes() {
 	r.Get("/plugins/:id/page/:page", reqSignedIn, Index)
 
 	r.Get("/d/:uid/:slug", reqSignedIn, Index)
+	r.Get("/d/:uid", reqSignedIn, Index)
 	r.Get("/dashboard/db/:slug", reqSignedIn, redirectFromLegacyDashboardUrl, Index)
 	r.Get("/dashboard/script/*", reqSignedIn, Index)
 	r.Get("/dashboard-solo/snapshot/*", Index)
@@ -150,11 +151,11 @@ func (hs *HttpServer) registerRoutes() {
 		apiRoute.Group("/teams", func(teamsRoute RouteRegister) {
 			teamsRoute.Get("/:teamId", wrap(GetTeamById))
 			teamsRoute.Get("/search", wrap(SearchTeams))
-			teamsRoute.Post("/", quota("teams"), bind(m.CreateTeamCommand{}), wrap(CreateTeam))
+			teamsRoute.Post("/", bind(m.CreateTeamCommand{}), wrap(CreateTeam))
 			teamsRoute.Put("/:teamId", bind(m.UpdateTeamCommand{}), wrap(UpdateTeam))
 			teamsRoute.Delete("/:teamId", wrap(DeleteTeamById))
 			teamsRoute.Get("/:teamId/members", wrap(GetTeamMembers))
-			teamsRoute.Post("/:teamId/members", quota("teams"), bind(m.AddTeamMemberCommand{}), wrap(AddTeamMember))
+			teamsRoute.Post("/:teamId/members", bind(m.AddTeamMemberCommand{}), wrap(AddTeamMember))
 			teamsRoute.Delete("/:teamId/members/:userId", wrap(RemoveTeamMember))
 		}, reqOrgAdmin)
 

pkg/api/common_test.go

@@ -99,7 +99,7 @@ func setupScenarioContext(url string) *scenarioContext {
 	}))
 
 	sc.m.Use(middleware.GetContextHandler())
-	sc.m.Use(middleware.Sessioner(&session.Options{}))
+	sc.m.Use(middleware.Sessioner(&session.Options{}, 0))
 
 	return sc
 }

pkg/api/http_server.go

@@ -175,7 +175,7 @@ func (hs *HttpServer) newMacaron() *macaron.Macaron {
 	m.Use(hs.healthHandler)
 	m.Use(hs.metricsEndpoint)
 	m.Use(middleware.GetContextHandler())
-	m.Use(middleware.Sessioner(&setting.SessionOptions))
+	m.Use(middleware.Sessioner(&setting.SessionOptions, setting.SessionConnMaxLifetime))
 	m.Use(middleware.OrgRedirect())
 
 	// needs to be after context handler

pkg/api/index.go

@@ -74,7 +74,7 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) {
 	}
 
 	if setting.DisableGravatar {
-		data.User.GravatarUrl = setting.AppSubUrl + "/public/img/transparent.png"
+		data.User.GravatarUrl = setting.AppSubUrl + "/public/img/user_profile.png"
 	}
 
 	if len(data.User.Name) == 0 {

pkg/components/renderer/renderer.go

@@ -72,7 +72,9 @@ func RenderToPng(params *RenderOpts) (string, error) {
 		localDomain = setting.HttpAddr
 	}
 
-	url := fmt.Sprintf("%s://%s:%s/%s", setting.Protocol, localDomain, setting.HttpPort, params.Path)
+	// &render=1 signals to the legacy redirect layer to
+	// avoid redirect these requests.
+	url := fmt.Sprintf("%s://%s:%s/%s&render=1", setting.Protocol, localDomain, setting.HttpPort, params.Path)
 
 	binPath, _ := filepath.Abs(filepath.Join(setting.PhantomDir, executable))
 	scriptPath, _ := filepath.Abs(filepath.Join(setting.PhantomDir, "render.js"))

pkg/middleware/dashboard_redirect.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/bus"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 	"gopkg.in/macaron.v1"
 )
 
@@ -36,9 +37,14 @@ func RedirectFromLegacyDashboardUrl() macaron.Handler {
 func RedirectFromLegacyDashboardSoloUrl() macaron.Handler {
 	return func(c *Context) {
 		slug := c.Params("slug")
+		renderRequest := c.QueryBool("render")
 
 		if slug != "" {
 			if url, err := getDashboardUrlBySlug(c.OrgId, slug); err == nil {
+				if renderRequest && strings.Contains(url, setting.AppSubUrl) {
+					url = strings.Replace(url, setting.AppSubUrl, "", 1)
+				}
+
 				url = strings.Replace(url, "/d/", "/d-solo/", 1)
 				url = fmt.Sprintf("%s?%s", url, c.Req.URL.RawQuery)
 				c.Redirect(url, 301)

pkg/middleware/middleware_test.go

@@ -337,7 +337,7 @@ func middlewareScenario(desc string, fn scenarioFunc) {
 		sc.m.Use(GetContextHandler())
 		// mock out gc goroutine
 		startSessionGC = func() {}
-		sc.m.Use(Sessioner(&session.Options{}))
+		sc.m.Use(Sessioner(&session.Options{}, 0))
 		sc.m.Use(OrgRedirect())
 		sc.m.Use(AddDefaultResponseHeaders())
 

pkg/middleware/mysql.go

@@ -0,0 +1,218 @@
+// Copyright 2013 Beego Authors
+// Copyright 2014 The Macaron Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package middleware
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+
+	"github.com/go-macaron/session"
+)
+
+// MysqlStore represents a mysql session store implementation.
+type MysqlStore struct {
+	c    *sql.DB
+	sid  string
+	lock sync.RWMutex
+	data map[interface{}]interface{}
+}
+
+// NewMysqlStore creates and returns a mysql session store.
+func NewMysqlStore(c *sql.DB, sid string, kv map[interface{}]interface{}) *MysqlStore {
+	return &MysqlStore{
+		c:    c,
+		sid:  sid,
+		data: kv,
+	}
+}
+
+// Set sets value to given key in session.
+func (s *MysqlStore) Set(key, val interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data[key] = val
+	return nil
+}
+
+// Get gets value by given key in session.
+func (s *MysqlStore) Get(key interface{}) interface{} {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	return s.data[key]
+}
+
+// Delete delete a key from session.
+func (s *MysqlStore) Delete(key interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	delete(s.data, key)
+	return nil
+}
+
+// ID returns current session ID.
+func (s *MysqlStore) ID() string {
+	return s.sid
+}
+
+// Release releases resource and save data to provider.
+func (s *MysqlStore) Release() error {
+	data, err := session.EncodeGob(s.data)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.c.Exec("UPDATE session SET data=?, expiry=? WHERE `key`=?",
+		data, time.Now().Unix(), s.sid)
+	return err
+}
+
+// Flush deletes all session data.
+func (s *MysqlStore) Flush() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data = make(map[interface{}]interface{})
+	return nil
+}
+
+// MysqlProvider represents a mysql session provider implementation.
+type MysqlProvider struct {
+	c      *sql.DB
+	expire int64
+}
+
+// Init initializes mysql session provider.
+// connStr: username:password@protocol(address)/dbname?param=value
+func (p *MysqlProvider) Init(expire int64, connStr string) (err error) {
+	p.expire = expire
+
+	p.c, err = sql.Open("mysql", connStr)
+	p.c.SetConnMaxLifetime(time.Second * time.Duration(sessionConnMaxLifetime))
+	if err != nil {
+		return err
+	}
+	return p.c.Ping()
+}
+
+// Read returns raw session store by session ID.
+func (p *MysqlProvider) Read(sid string) (session.RawStore, error) {
+	var data []byte
+	err := p.c.QueryRow("SELECT data FROM session WHERE `key`=?", sid).Scan(&data)
+	if err == sql.ErrNoRows {
+		_, err = p.c.Exec("INSERT INTO session(`key`,data,expiry) VALUES(?,?,?)",
+			sid, "", time.Now().Unix())
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var kv map[interface{}]interface{}
+	if len(data) == 0 {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob(data)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return NewMysqlStore(p.c, sid, kv), nil
+}
+
+// Exist returns true if session with given ID exists.
+func (p *MysqlProvider) Exist(sid string) bool {
+	exists, err := p.queryExists(sid)
+
+	if err != nil {
+		exists, err = p.queryExists(sid)
+	}
+
+	if err != nil {
+		log.Printf("session/mysql: error checking if session exists: %v", err)
+		return false
+	}
+
+	return exists
+}
+
+func (p *MysqlProvider) queryExists(sid string) (bool, error) {
+	var data []byte
+	err := p.c.QueryRow("SELECT data FROM session WHERE `key`=?", sid).Scan(&data)
+
+	if err != nil && err != sql.ErrNoRows {
+		return false, err
+	}
+
+	return err != sql.ErrNoRows, nil
+}
+
+// Destory deletes a session by session ID.
+func (p *MysqlProvider) Destory(sid string) error {
+	_, err := p.c.Exec("DELETE FROM session WHERE `key`=?", sid)
+	return err
+}
+
+// Regenerate regenerates a session store from old session ID to new one.
+func (p *MysqlProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) {
+	if p.Exist(sid) {
+		return nil, fmt.Errorf("new sid '%s' already exists", sid)
+	}
+
+	if !p.Exist(oldsid) {
+		if _, err = p.c.Exec("INSERT INTO session(`key`,data,expiry) VALUES(?,?,?)",
+			oldsid, "", time.Now().Unix()); err != nil {
+			return nil, err
+		}
+	}
+
+	if _, err = p.c.Exec("UPDATE session SET `key`=? WHERE `key`=?", sid, oldsid); err != nil {
+		return nil, err
+	}
+
+	return p.Read(sid)
+}
+
+// Count counts and returns number of sessions.
+func (p *MysqlProvider) Count() (total int) {
+	if err := p.c.QueryRow("SELECT COUNT(*) AS NUM FROM session").Scan(&total); err != nil {
+		panic("session/mysql: error counting records: " + err.Error())
+	}
+	return total
+}
+
+// GC calls GC to clean expired sessions.
+func (p *MysqlProvider) GC() {
+	var err error
+	if _, err = p.c.Exec("DELETE FROM session WHERE  expiry + ? <= UNIX_TIMESTAMP(NOW())", p.expire); err != nil {
+		_, err = p.c.Exec("DELETE FROM session WHERE  expiry + ? <= UNIX_TIMESTAMP(NOW())", p.expire)
+	}
+
+	if err != nil {
+		log.Printf("session/mysql: error garbage collecting: %v", err)
+	}
+}
+
+func init() {
+	session.Register("mysql", &MysqlProvider{})
+}

pkg/middleware/recovery_test.go

@@ -61,7 +61,7 @@ func recoveryScenario(desc string, url string, fn scenarioFunc) {
 		sc.m.Use(GetContextHandler())
 		// mock out gc goroutine
 		startSessionGC = func() {}
-		sc.m.Use(Sessioner(&session.Options{}))
+		sc.m.Use(Sessioner(&session.Options{}, 0))
 		sc.m.Use(OrgRedirect())
 		sc.m.Use(AddDefaultResponseHeaders())
 

pkg/middleware/session.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/go-macaron/session"
 	_ "github.com/go-macaron/session/memcache"
-	_ "github.com/go-macaron/session/mysql"
 	_ "github.com/go-macaron/session/postgres"
 	_ "github.com/go-macaron/session/redis"
 	"github.com/grafana/grafana/pkg/log"
@@ -25,6 +24,7 @@ var sessionOptions *session.Options
 var startSessionGC func()
 var getSessionCount func() int
 var sessionLogger = log.New("session")
+var sessionConnMaxLifetime int64
 
 func init() {
 	startSessionGC = func() {
@@ -63,9 +63,10 @@ func prepareOptions(opt *session.Options) *session.Options {
 	return opt
 }
 
-func Sessioner(options *session.Options) macaron.Handler {
+func Sessioner(options *session.Options, connMaxLifetime int64) macaron.Handler {
 	var err error
 	sessionOptions = prepareOptions(options)
+	sessionConnMaxLifetime = connMaxLifetime
 	sessionManager, err = session.NewManager(options.Provider, *options)
 	if err != nil {
 		panic(err)
@@ -113,6 +114,18 @@ type SessionWrapper struct {
 }
 
 func (s *SessionWrapper) Start(c *Context) error {
+	// See https://github.com/grafana/grafana/issues/11155 for details on why
+	// a recover and retry is needed
+	defer func() error {
+		if err := recover(); err != nil {
+			var retryErr error
+			s.session, retryErr = s.manager.Start(c.Context)
+			return retryErr
+		}
+
+		return nil
+	}()
+
 	var err error
 	s.session, err = s.manager.Start(c.Context)
 	return err

pkg/models/dashboards.go

@@ -209,14 +209,14 @@ func GetDashboardFolderUrl(isFolder bool, uid string, slug string) string {
 	return GetDashboardUrl(uid, slug)
 }
 
-// Return the html url for a dashboard
+// GetDashboardUrl return the html url for a dashboard
 func GetDashboardUrl(uid string, slug string) string {
 	return fmt.Sprintf("%s/d/%s/%s", setting.AppSubUrl, uid, slug)
 }
 
-// Return the full url for a dashboard
+// GetFullDashboardUrl return the full url for a dashboard
 func GetFullDashboardUrl(uid string, slug string) string {
-	return fmt.Sprintf("%s%s", setting.AppUrl, GetDashboardUrl(uid, slug))
+	return fmt.Sprintf("%sd/%s/%s", setting.AppUrl, uid, slug)
 }
 
 // GetFolderUrl return the html url for a folder

pkg/models/dashboards_test.go

@@ -4,11 +4,24 @@ import (
 	"testing"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/setting"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
 func TestDashboardModel(t *testing.T) {
 
+	Convey("Generate full dashboard url", t, func() {
+		setting.AppUrl = "http://grafana.local/"
+		fullUrl := GetFullDashboardUrl("uid", "my-dashboard")
+		So(fullUrl, ShouldEqual, "http://grafana.local/d/uid/my-dashboard")
+	})
+
+	Convey("Generate relative dashboard url", t, func() {
+		setting.AppUrl = ""
+		fullUrl := GetDashboardUrl("uid", "my-dashboard")
+		So(fullUrl, ShouldEqual, "/d/uid/my-dashboard")
+	})
+
 	Convey("When generating slug", t, func() {
 		dashboard := NewDashboard("Grafana Play Home")
 		dashboard.UpdateSlug()

pkg/plugins/dashboard_importer.go

@@ -80,7 +80,7 @@ func ImportDashboard(cmd *ImportDashboardCommand) error {
 		User:      cmd.User,
 	}
 
-	savedDash, err := dashboards.NewService().SaveDashboard(dto)
+	savedDash, err := dashboards.NewService().ImportDashboard(dto)
 
 	if err != nil {
 		return err

pkg/services/alerting/commands.go

@@ -13,11 +13,7 @@ func init() {
 func validateDashboardAlerts(cmd *m.ValidateDashboardAlertsCommand) error {
 	extractor := NewDashAlertExtractor(cmd.Dashboard, cmd.OrgId)
 
-	if _, err := extractor.GetAlerts(); err != nil {
+	return extractor.ValidateAlerts()
-		return err
-	}
-
-	return nil
 }
 
 func updateDashboardAlerts(cmd *m.UpdateDashboardAlertsCommand) error {
@@ -29,15 +25,12 @@ func updateDashboardAlerts(cmd *m.UpdateDashboardAlertsCommand) error {
 
 	extractor := NewDashAlertExtractor(cmd.Dashboard, cmd.OrgId)
 
-	if alerts, err := extractor.GetAlerts(); err != nil {
+	alerts, err := extractor.GetAlerts()
-		return err
+	if err != nil {
-	} else {
-		saveAlerts.Alerts = alerts
-	}
-
-	if err := bus.Dispatch(&saveAlerts); err != nil {
 		return err
 	}
 
-	return nil
+	saveAlerts.Alerts = alerts
+
+	return bus.Dispatch(&saveAlerts)
 }

pkg/services/alerting/extractor.go

@@ -11,76 +11,93 @@ import (
 	m "github.com/grafana/grafana/pkg/models"
 )
 
+// DashAlertExtractor extracts alerts from the dashboard json
 type DashAlertExtractor struct {
 	Dash  *m.Dashboard
-	OrgId int64
+	OrgID int64
 	log   log.Logger
 }
 
-func NewDashAlertExtractor(dash *m.Dashboard, orgId int64) *DashAlertExtractor {
+// NewDashAlertExtractor returns a new DashAlertExtractor
+func NewDashAlertExtractor(dash *m.Dashboard, orgID int64) *DashAlertExtractor {
 	return &DashAlertExtractor{
 		Dash:  dash,
-		OrgId: orgId,
+		OrgID: orgID,
 		log:   log.New("alerting.extractor"),
 	}
 }
 
-func (e *DashAlertExtractor) lookupDatasourceId(dsName string) (*m.DataSource, error) {
+func (e *DashAlertExtractor) lookupDatasourceID(dsName string) (*m.DataSource, error) {
 	if dsName == "" {
-		query := &m.GetDataSourcesQuery{OrgId: e.OrgId}
+		query := &m.GetDataSourcesQuery{OrgId: e.OrgID}
 		if err := bus.Dispatch(query); err != nil {
 			return nil, err
-		} else {
+		}
-			for _, ds := range query.Result {
+
-				if ds.IsDefault {
+		for _, ds := range query.Result {
-					return ds, nil
+			if ds.IsDefault {
-				}
+				return ds, nil
 			}
 		}
 	} else {
-		query := &m.GetDataSourceByNameQuery{Name: dsName, OrgId: e.OrgId}
+		query := &m.GetDataSourceByNameQuery{Name: dsName, OrgId: e.OrgID}
 		if err := bus.Dispatch(query); err != nil {
 			return nil, err
-		} else {
-			return query.Result, nil
 		}
+
+		return query.Result, nil
 	}
 
 	return nil, errors.New("Could not find datasource id for " + dsName)
 }
 
-func findPanelQueryByRefId(panel *simplejson.Json, refId string) *simplejson.Json {
+func findPanelQueryByRefID(panel *simplejson.Json, refID string) *simplejson.Json {
 	for _, targetsObj := range panel.Get("targets").MustArray() {
 		target := simplejson.NewFromAny(targetsObj)
 
-		if target.Get("refId").MustString() == refId {
+		if target.Get("refId").MustString() == refID {
 			return target
 		}
 	}
 	return nil
 }
 
-func copyJson(in *simplejson.Json) (*simplejson.Json, error) {
+func copyJSON(in *simplejson.Json) (*simplejson.Json, error) {
-	rawJson, err := in.MarshalJSON()
+	rawJSON, err := in.MarshalJSON()
 	if err != nil {
 		return nil, err
 	}
 
-	return simplejson.NewJson(rawJson)
+	return simplejson.NewJson(rawJSON)
 }
 
-func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json) ([]*m.Alert, error) {
+func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json, validateAlertFunc func(*m.Alert) bool) ([]*m.Alert, error) {
 	alerts := make([]*m.Alert, 0)
 
 	for _, panelObj := range jsonWithPanels.Get("panels").MustArray() {
 		panel := simplejson.NewFromAny(panelObj)
+
+		collapsedJSON, collapsed := panel.CheckGet("collapsed")
+		// check if the panel is collapsed
+		if collapsed && collapsedJSON.MustBool() {
+
+			// extract alerts from sub panels for collapsed panels
+			als, err := e.getAlertFromPanels(panel, validateAlertFunc)
+			if err != nil {
+				return nil, err
+			}
+
+			alerts = append(alerts, als...)
+			continue
+		}
+
 		jsonAlert, hasAlert := panel.CheckGet("alert")
 
 		if !hasAlert {
 			continue
 		}
 
-		panelId, err := panel.Get("id").Int64()
+		panelID, err := panel.Get("id").Int64()
 		if err != nil {
 			return nil, fmt.Errorf("panel id is required. err %v", err)
 		}
@@ -98,8 +115,8 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json)
 
 		alert := &m.Alert{
 			DashboardId: e.Dash.Id,
-			OrgId:       e.OrgId,
+			OrgId:       e.OrgID,
-			PanelId:     panelId,
+			PanelId:     panelID,
 			Id:          jsonAlert.Get("id").MustInt64(),
 			Name:        jsonAlert.Get("name").MustString(),
 			Handler:     jsonAlert.Get("handler").MustInt64(),
@@ -111,11 +128,11 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json)
 			jsonCondition := simplejson.NewFromAny(condition)
 
 			jsonQuery := jsonCondition.Get("query")
-			queryRefId := jsonQuery.Get("params").MustArray()[0].(string)
+			queryRefID := jsonQuery.Get("params").MustArray()[0].(string)
-			panelQuery := findPanelQueryByRefId(panel, queryRefId)
+			panelQuery := findPanelQueryByRefID(panel, queryRefID)
 
 			if panelQuery == nil {
-				reason := fmt.Sprintf("Alert on PanelId: %v refers to query(%s) that cannot be found", alert.PanelId, queryRefId)
+				reason := fmt.Sprintf("Alert on PanelId: %v refers to query(%s) that cannot be found", alert.PanelId, queryRefID)
 				return nil, ValidationError{Reason: reason}
 			}
 
@@ -126,12 +143,13 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json)
 				dsName = panel.Get("datasource").MustString()
 			}
 
-			if datasource, err := e.lookupDatasourceId(dsName); err != nil {
+			datasource, err := e.lookupDatasourceID(dsName)
+			if err != nil {
 				return nil, err
-			} else {
-				jsonQuery.SetPath([]string{"datasourceId"}, datasource.Id)
 			}
 
+			jsonQuery.SetPath([]string{"datasourceId"}, datasource.Id)
+
 			if interval, err := panel.Get("interval").String(); err == nil {
 				panelQuery.Set("interval", interval)
 			}
@@ -143,20 +161,32 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json)
 
 		// validate
 		_, err = NewRuleFromDBAlert(alert)
-		if err == nil && alert.ValidToSave() {
+		if err != nil {
-			alerts = append(alerts, alert)
-		} else {
 			return nil, err
 		}
+
+		if !validateAlertFunc(alert) {
+			e.log.Debug("Invalid Alert Data. Dashboard, Org or Panel ID is not correct", "alertName", alert.Name, "panelId", alert.PanelId)
+			return nil, m.ErrDashboardContainsInvalidAlertData
+		}
+
+		alerts = append(alerts, alert)
 	}
 
 	return alerts, nil
 }
 
-func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
+func validateAlertRule(alert *m.Alert) bool {
-	e.log.Debug("GetAlerts")
+	return alert.ValidToSave()
+}
 
-	dashboardJson, err := copyJson(e.Dash.Data)
+// GetAlerts extracts alerts from the dashboard json and does full validation on the alert json data
+func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
+	return e.extractAlerts(validateAlertRule)
+}
+
+func (e *DashAlertExtractor) extractAlerts(validateFunc func(alert *m.Alert) bool) ([]*m.Alert, error) {
+	dashboardJSON, err := copyJSON(e.Dash.Data)
 	if err != nil {
 		return nil, err
 	}
@@ -165,11 +195,11 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
 
 	// We extract alerts from rows to be backwards compatible
 	// with the old dashboard json model.
-	rows := dashboardJson.Get("rows").MustArray()
+	rows := dashboardJSON.Get("rows").MustArray()
 	if len(rows) > 0 {
 		for _, rowObj := range rows {
 			row := simplejson.NewFromAny(rowObj)
-			a, err := e.GetAlertFromPanels(row)
+			a, err := e.getAlertFromPanels(row, validateFunc)
 			if err != nil {
 				return nil, err
 			}
@@ -177,7 +207,7 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
 			alerts = append(alerts, a...)
 		}
 	} else {
-		a, err := e.GetAlertFromPanels(dashboardJson)
+		a, err := e.getAlertFromPanels(dashboardJSON, validateFunc)
 		if err != nil {
 			return nil, err
 		}
@@ -188,3 +218,10 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
 	e.log.Debug("Extracted alerts from dashboard", "alertCount", len(alerts))
 	return alerts, nil
 }
+
+// ValidateAlerts validates alerts in the dashboard json but does not require a valid dashboard id
+// in the first validation pass
+func (e *DashAlertExtractor) ValidateAlerts() error {
+	_, err := e.extractAlerts(func(alert *m.Alert) bool { return alert.OrgId != 0 && alert.PanelId != 0 })
+	return err
+}

pkg/services/alerting/extractor_test.go

@@ -22,6 +22,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		defaultDs := &m.DataSource{Id: 12, OrgId: 1, Name: "I am default", IsDefault: true}
 		graphite2Ds := &m.DataSource{Id: 15, OrgId: 1, Name: "graphite2"}
 		influxDBDs := &m.DataSource{Id: 16, OrgId: 1, Name: "InfluxDB"}
+		prom := &m.DataSource{Id: 17, OrgId: 1, Name: "Prometheus"}
 
 		bus.AddHandler("test", func(query *m.GetDataSourcesQuery) error {
 			query.Result = []*m.DataSource{defaultDs, graphite2Ds}
@@ -38,6 +39,10 @@ func TestAlertRuleExtraction(t *testing.T) {
 			if query.Name == influxDBDs.Name {
 				query.Result = influxDBDs
 			}
+			if query.Name == prom.Name {
+				query.Result = prom
+			}
+
 			return nil
 		})
 
@@ -150,6 +155,22 @@ func TestAlertRuleExtraction(t *testing.T) {
 			})
 		})
 
+		Convey("Panel with id set to zero should return error", func() {
+			panelWithIdZero, err := ioutil.ReadFile("./test-data/panel-with-id-0.json")
+			So(err, ShouldBeNil)
+
+			dashJson, err := simplejson.NewJson([]byte(panelWithIdZero))
+			So(err, ShouldBeNil)
+			dash := m.NewDashboardFromJson(dashJson)
+			extractor := NewDashAlertExtractor(dash, 1)
+
+			_, err = extractor.GetAlerts()
+
+			Convey("panel with id 0 should return error", func() {
+				So(err, ShouldNotBeNil)
+			})
+		})
+
 		Convey("Parse alerts from dashboard without rows", func() {
 			json, err := ioutil.ReadFile("./test-data/v5-dashboard.json")
 			So(err, ShouldBeNil)
@@ -198,5 +219,47 @@ func TestAlertRuleExtraction(t *testing.T) {
 				}
 			})
 		})
+
+		Convey("Should be able to extract collapsed panels", func() {
+			json, err := ioutil.ReadFile("./test-data/collapsed-panels.json")
+			So(err, ShouldBeNil)
+
+			dashJson, err := simplejson.NewJson(json)
+			So(err, ShouldBeNil)
+
+			dash := m.NewDashboardFromJson(dashJson)
+			extractor := NewDashAlertExtractor(dash, 1)
+
+			alerts, err := extractor.GetAlerts()
+
+			Convey("Get rules without error", func() {
+				So(err, ShouldBeNil)
+			})
+
+			Convey("should be able to extract collapsed alerts", func() {
+				So(len(alerts), ShouldEqual, 4)
+			})
+		})
+
+		Convey("Parse and validate dashboard without id and containing an alert", func() {
+			json, err := ioutil.ReadFile("./test-data/dash-without-id.json")
+			So(err, ShouldBeNil)
+
+			dashJSON, err := simplejson.NewJson(json)
+			So(err, ShouldBeNil)
+			dash := m.NewDashboardFromJson(dashJSON)
+			extractor := NewDashAlertExtractor(dash, 1)
+
+			err = extractor.ValidateAlerts()
+
+			Convey("Should validate without error", func() {
+				So(err, ShouldBeNil)
+			})
+
+			Convey("Should fail on save", func() {
+				_, err := extractor.GetAlerts()
+				So(err, ShouldEqual, m.ErrDashboardContainsInvalidAlertData)
+			})
+		})
 	})
 }

pkg/services/alerting/notifiers/base.go

@@ -15,7 +15,11 @@ type NotifierBase struct {
 }
 
 func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model *simplejson.Json) NotifierBase {
-	uploadImage := model.Get("uploadImage").MustBool(false)
+	uploadImage := true
+	value, exist := model.CheckGet("uploadImage")
+	if exist {
+		uploadImage = value.MustBool()
+	}
 
 	return NotifierBase{
 		Id:          id,

pkg/services/alerting/notifiers/base_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/grafana/grafana/pkg/components/simplejson"
 	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting"
 	. "github.com/smartystreets/goconvey/convey"
@@ -11,6 +12,29 @@ import (
 
 func TestBaseNotifier(t *testing.T) {
 	Convey("Base notifier tests", t, func() {
+		Convey("default constructor for notifiers", func() {
+			bJson := simplejson.New()
+
+			Convey("can parse false value", func() {
+				bJson.Set("uploadImage", false)
+
+				base := NewNotifierBase(1, false, "name", "email", bJson)
+				So(base.UploadImage, ShouldBeFalse)
+			})
+
+			Convey("can parse true value", func() {
+				bJson.Set("uploadImage", true)
+
+				base := NewNotifierBase(1, false, "name", "email", bJson)
+				So(base.UploadImage, ShouldBeTrue)
+			})
+
+			Convey("default value should be true for backwards compatibility", func() {
+				base := NewNotifierBase(1, false, "name", "email", bJson)
+				So(base.UploadImage, ShouldBeTrue)
+			})
+		})
+
 		Convey("should notify", func() {
 			Convey("pending -> ok", func() {
 				context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{

pkg/services/alerting/notifiers/telegram.go

@@ -12,6 +12,10 @@ import (
 	"os"
 )
 
+const (
+	captionLengthLimit = 200
+)
+
 var (
 	telegramApiUrl string = "https://api.telegram.org/bot%s/%s"
 )
@@ -82,88 +86,81 @@ func NewTelegramNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 }
 
 func (this *TelegramNotifier) buildMessage(evalContext *alerting.EvalContext, sendImageInline bool) *m.SendWebhookSync {
-	var imageFile *os.File
-	var err error
-
 	if sendImageInline {
-		imageFile, err = os.Open(evalContext.ImageOnDiskPath)
+		cmd, err := this.buildMessageInlineImage(evalContext)
-		defer imageFile.Close()
+		if err == nil {
-		if err != nil {
+			return cmd
-			sendImageInline = false // fall back to text message
+		} else {
+			this.log.Error("Could not generate Telegram message with inline image.", "err", err)
 		}
 	}
 
-	message := ""
+	return this.buildMessageLinkedImage(evalContext)
+}
 
-	if sendImageInline {
+func (this *TelegramNotifier) buildMessageLinkedImage(evalContext *alerting.EvalContext) *m.SendWebhookSync {
-		// Telegram's API does not allow HTML formatting for image captions.
+	message := fmt.Sprintf("<b>%s</b>\nState: %s\nMessage: %s\n", evalContext.GetNotificationTitle(), evalContext.Rule.Name, evalContext.Rule.Message)
-		message = fmt.Sprintf("%s\nState: %s\nMessage: %s\n", evalContext.GetNotificationTitle(), evalContext.Rule.Name, evalContext.Rule.Message)
-	} else {
-		message = fmt.Sprintf("<b>%s</b>\nState: %s\nMessage: %s\n", evalContext.GetNotificationTitle(), evalContext.Rule.Name, evalContext.Rule.Message)
-	}
 
 	ruleUrl, err := evalContext.GetRuleUrl()
 	if err == nil {
 		message = message + fmt.Sprintf("URL: %s\n", ruleUrl)
 	}
 
-	if !sendImageInline {
+	if evalContext.ImagePublicUrl != "" {
-		// only attach this if we are not sending it inline.
+		message = message + fmt.Sprintf("Image: %s\n", evalContext.ImagePublicUrl)
-		if evalContext.ImagePublicUrl != "" {
-			message = message + fmt.Sprintf("Image: %s\n", evalContext.ImagePublicUrl)
-		}
-	}
-
-	metrics := ""
-	fieldLimitCount := 4
-	for index, evt := range evalContext.EvalMatches {
-		metrics += fmt.Sprintf("\n%s: %s", evt.Metric, evt.Value)
-		if index > fieldLimitCount {
-			break
-		}
 	}
 
+	metrics := generateMetricsMessage(evalContext)
 	if metrics != "" {
-		if sendImageInline {
+		message = message + fmt.Sprintf("\n<i>Metrics:</i>%s", metrics)
-			// Telegram's API does not allow HTML formatting for image captions.
-			message = message + fmt.Sprintf("\nMetrics:%s", metrics)
-		} else {
-			message = message + fmt.Sprintf("\n<i>Metrics:</i>%s", metrics)
-		}
 	}
 
-	var body bytes.Buffer
+	cmd := this.generateTelegramCmd(message, "text", "sendMessage", func(w *multipart.Writer) {
+		fw, _ := w.CreateFormField("parse_mode")
+		fw.Write([]byte("html"))
+	})
+	return cmd
+}
 
+func (this *TelegramNotifier) buildMessageInlineImage(evalContext *alerting.EvalContext) (*m.SendWebhookSync, error) {
+	var imageFile *os.File
+	var err error
+
+	imageFile, err = os.Open(evalContext.ImageOnDiskPath)
+	defer imageFile.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	ruleUrl, err := evalContext.GetRuleUrl()
+
+	metrics := generateMetricsMessage(evalContext)
+	message := generateImageCaption(evalContext, ruleUrl, metrics)
+
+	cmd := this.generateTelegramCmd(message, "caption", "sendPhoto", func(w *multipart.Writer) {
+		fw, _ := w.CreateFormFile("photo", evalContext.ImageOnDiskPath)
+		io.Copy(fw, imageFile)
+	})
+	return cmd, nil
+}
+
+func (this *TelegramNotifier) generateTelegramCmd(message string, messageField string, apiAction string, extraConf func(writer *multipart.Writer)) *m.SendWebhookSync {
+	var body bytes.Buffer
 	w := multipart.NewWriter(&body)
+
 	fw, _ := w.CreateFormField("chat_id")
 	fw.Write([]byte(this.ChatID))
 
-	if sendImageInline {
+	fw, _ = w.CreateFormField(messageField)
-		fw, _ = w.CreateFormField("caption")
+	fw.Write([]byte(message))
-		fw.Write([]byte(message))
 
-		fw, _ = w.CreateFormFile("photo", evalContext.ImageOnDiskPath)
+	extraConf(w)
-		io.Copy(fw, imageFile)
-	} else {
-		fw, _ = w.CreateFormField("text")
-		fw.Write([]byte(message))
-
-		fw, _ = w.CreateFormField("parse_mode")
-		fw.Write([]byte("html"))
-	}
 
 	w.Close()
 
-	apiMethod := ""
+	this.log.Info("Sending telegram notification", "chat_id", this.ChatID, "bot_token", this.BotToken, "apiAction", apiAction)
-	if sendImageInline {
+	url := fmt.Sprintf(telegramApiUrl, this.BotToken, apiAction)
-		this.log.Info("Sending telegram image notification", "photo", evalContext.ImageOnDiskPath, "chat_id", this.ChatID, "bot_token", this.BotToken)
-		apiMethod = "sendPhoto"
-	} else {
-		this.log.Info("Sending telegram text notification", "chat_id", this.ChatID, "bot_token", this.BotToken)
-		apiMethod = "sendMessage"
-	}
 
-	url := fmt.Sprintf(telegramApiUrl, this.BotToken, apiMethod)
 	cmd := &m.SendWebhookSync{
 		Url:        url,
 		Body:       body.String(),
@@ -175,6 +172,50 @@ func (this *TelegramNotifier) buildMessage(evalContext *alerting.EvalContext, se
 	return cmd
 }
 
+func generateMetricsMessage(evalContext *alerting.EvalContext) string {
+	metrics := ""
+	fieldLimitCount := 4
+	for index, evt := range evalContext.EvalMatches {
+		metrics += fmt.Sprintf("\n%s: %s", evt.Metric, evt.Value)
+		if index > fieldLimitCount {
+			break
+		}
+	}
+	return metrics
+}
+
+func generateImageCaption(evalContext *alerting.EvalContext, ruleUrl string, metrics string) string {
+	message := evalContext.GetNotificationTitle()
+
+	if len(evalContext.Rule.Message) > 0 {
+		message = fmt.Sprintf("%s\nMessage: %s", message, evalContext.Rule.Message)
+	}
+
+	if len(message) > captionLengthLimit {
+		message = message[0:captionLengthLimit]
+
+	}
+
+	if len(ruleUrl) > 0 {
+		urlLine := fmt.Sprintf("\nURL: %s", ruleUrl)
+		message = appendIfPossible(message, urlLine, captionLengthLimit)
+	}
+
+	if metrics != "" {
+		metricsLines := fmt.Sprintf("\n\nMetrics:%s", metrics)
+		message = appendIfPossible(message, metricsLines, captionLengthLimit)
+	}
+
+	return message
+}
+func appendIfPossible(message string, extra string, sizeLimit int) string {
+	if len(extra)+len(message) <= sizeLimit {
+		return message + extra
+	}
+	log.Debug("Line too long for image caption.", "value", extra)
+	return message
+}
+
 func (this *TelegramNotifier) ShouldNotify(context *alerting.EvalContext) bool {
 	return defaultShouldNotify(context)
 }

pkg/services/alerting/notifiers/telegram_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
@@ -50,6 +51,71 @@ func TestTelegramNotifier(t *testing.T) {
 				So(telegramNotifier.ChatID, ShouldEqual, "-1234567890")
 			})
 
+			Convey("generateCaption should generate a message with all pertinent details", func() {
+				evalContext := alerting.NewEvalContext(nil, &alerting.Rule{
+					Name:    "This is an alarm",
+					Message: "Some kind of message.",
+					State:   m.AlertStateOK,
+				})
+
+				caption := generateImageCaption(evalContext, "http://grafa.url/abcdef", "")
+				So(len(caption), ShouldBeLessThanOrEqualTo, 200)
+				So(caption, ShouldContainSubstring, "Some kind of message.")
+				So(caption, ShouldContainSubstring, "[OK] This is an alarm")
+				So(caption, ShouldContainSubstring, "http://grafa.url/abcdef")
+			})
+
+			Convey("When generating a message", func() {
+
+				Convey("URL should be skipped if it's too long", func() {
+					evalContext := alerting.NewEvalContext(nil, &alerting.Rule{
+						Name:    "This is an alarm",
+						Message: "Some kind of message.",
+						State:   m.AlertStateOK,
+					})
+
+					caption := generateImageCaption(evalContext,
+						"http://grafa.url/abcdefaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+						"foo bar")
+					So(len(caption), ShouldBeLessThanOrEqualTo, 200)
+					So(caption, ShouldContainSubstring, "Some kind of message.")
+					So(caption, ShouldContainSubstring, "[OK] This is an alarm")
+					So(caption, ShouldContainSubstring, "foo bar")
+					So(caption, ShouldNotContainSubstring, "http")
+				})
+
+				Convey("Message should be trimmed if it's too long", func() {
+					evalContext := alerting.NewEvalContext(nil, &alerting.Rule{
+						Name:    "This is an alarm",
+						Message: "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I promise I will. Yes siree that's it.",
+						State:   m.AlertStateOK,
+					})
+
+					caption := generateImageCaption(evalContext,
+						"http://grafa.url/foo",
+						"")
+					So(len(caption), ShouldBeLessThanOrEqualTo, 200)
+					So(caption, ShouldContainSubstring, "[OK] This is an alarm")
+					So(caption, ShouldNotContainSubstring, "http")
+					So(caption, ShouldContainSubstring, "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I promise ")
+				})
+
+				Convey("Metrics should be skipped if they dont fit", func() {
+					evalContext := alerting.NewEvalContext(nil, &alerting.Rule{
+						Name:    "This is an alarm",
+						Message: "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I ",
+						State:   m.AlertStateOK,
+					})
+
+					caption := generateImageCaption(evalContext,
+						"http://grafa.url/foo",
+						"foo bar long song")
+					So(len(caption), ShouldBeLessThanOrEqualTo, 200)
+					So(caption, ShouldContainSubstring, "[OK] This is an alarm")
+					So(caption, ShouldNotContainSubstring, "http")
+					So(caption, ShouldNotContainSubstring, "foo bar")
+				})
+			})
 		})
 	})
 }

pkg/services/alerting/test-data/collapsed-panels.json

@@ -0,0 +1,597 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 127,
+  "links": [],
+  "panels": [
+    {
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 9,
+      "title": "Row title",
+      "type": "row"
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                200
+              ],
+              "type": "gt"
+            },
+            "operator": {
+              "type": "and"
+            },
+            "query": {
+              "params": [
+                "A",
+                "5m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "executionErrorState": "alerting",
+        "frequency": "10s",
+        "handler": 1,
+        "name": "Panel Title alert",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 10,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "go_goroutines",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 200
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Panel Title",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ]
+    },
+    {
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 14,
+      "limit": 10,
+      "links": [],
+      "onlyAlertsOnDashboard": true,
+      "show": "current",
+      "sortOrder": 1,
+      "stateFilter": [],
+      "title": "Panel Title",
+      "type": "alertlist"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 10
+      },
+      "id": 6,
+      "panels": [
+        {
+          "alert": {
+            "conditions": [
+              {
+                "evaluator": {
+                  "params": [
+                    200
+                  ],
+                  "type": "gt"
+                },
+                "operator": {
+                  "type": "and"
+                },
+                "query": {
+                  "params": [
+                    "A",
+                    "5m",
+                    "now"
+                  ]
+                },
+                "reducer": {
+                  "params": [],
+                  "type": "avg"
+                },
+                "type": "query"
+              }
+            ],
+            "executionErrorState": "alerting",
+            "frequency": "10s",
+            "handler": 1,
+            "name": "Panel 2 alert",
+            "noDataState": "no_data",
+            "notifications": []
+          },
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "Prometheus",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 11
+          },
+          "id": 11,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "go_goroutines",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "critical",
+              "fill": true,
+              "line": true,
+              "op": "gt",
+              "value": 200
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Panel 2",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "alert": {
+            "conditions": [
+              {
+                "evaluator": {
+                  "params": [
+                    200
+                  ],
+                  "type": "gt"
+                },
+                "operator": {
+                  "type": "and"
+                },
+                "query": {
+                  "params": [
+                    "A",
+                    "5m",
+                    "now"
+                  ]
+                },
+                "reducer": {
+                  "params": [],
+                  "type": "avg"
+                },
+                "type": "query"
+              }
+            ],
+            "executionErrorState": "alerting",
+            "frequency": "10s",
+            "handler": 1,
+            "name": "Panel 4 alert",
+            "noDataState": "no_data",
+            "notifications": []
+          },
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "Prometheus",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 11
+          },
+          "id": 15,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "go_goroutines",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "critical",
+              "fill": true,
+              "line": true,
+              "op": "gt",
+              "value": 200
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Panel 4",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "title": "Row title",
+      "type": "row"
+    },
+    {
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 11
+      },
+      "id": 4,
+      "title": "Row title",
+      "type": "row"
+    },
+    {
+      "alert": {
+        "conditions": [
+          {
+            "evaluator": {
+              "params": [
+                200
+              ],
+              "type": "gt"
+            },
+            "operator": {
+              "type": "and"
+            },
+            "query": {
+              "params": [
+                "A",
+                "5m",
+                "now"
+              ]
+            },
+            "reducer": {
+              "params": [],
+              "type": "avg"
+            },
+            "type": "query"
+          }
+        ],
+        "executionErrorState": "alerting",
+        "frequency": "10s",
+        "handler": 1,
+        "name": "Panel 3 alert",
+        "noDataState": "no_data",
+        "notifications": []
+      },
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "id": 12,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "go_goroutines",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 200
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Panel 3",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 16,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "New dashboard Copy",
+  "uid": "6v5pg36zk",
+  "version": 17
+}

pkg/services/alerting/test-data/dash-without-id.json

@@ -0,0 +1,281 @@
+{
+    "title": "Influxdb",
+    "tags": [
+      "apa"
+    ],
+    "style": "dark",
+    "timezone": "browser",
+    "editable": true,
+    "hideControls": false,
+    "sharedCrosshair": false,
+    "rows": [
+      {
+        "collapse": false,
+        "editable": true,
+        "height": "450px",
+        "panels": [
+          {
+            "alert": {
+              "conditions": [
+                {
+                  "evaluator": {
+                    "params": [
+                      10
+                    ],
+                    "type": "gt"
+                  },
+                  "query": {
+                    "params": [
+                      "B",
+                      "5m",
+                      "now"
+                    ]
+                  },
+                  "reducer": {
+                    "params": [],
+                    "type": "avg"
+                  },
+                  "type": "query"
+                }
+              ],
+              "frequency": "3s",
+              "handler": 1,
+              "name": "Influxdb",
+              "noDataState": "no_data",
+              "notifications": [
+                {
+                  "id": 6
+                }
+              ]
+            },
+            "alerting": {},
+            "aliasColors": {
+              "logins.count.count": "#890F02"
+            },
+            "bars": false,
+            "datasource": "InfluxDB",
+            "editable": true,
+            "error": false,
+            "fill": 1,
+            "grid": {},
+            "id": 1,
+            "interval": ">10s",
+            "isNew": true,
+            "legend": {
+              "avg": false,
+              "current": false,
+              "max": false,
+              "min": false,
+              "show": true,
+              "total": false,
+              "values": false
+            },
+            "lines": true,
+            "linewidth": 2,
+            "links": [],
+            "nullPointMode": "connected",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "span": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "datacenter"
+                    ],
+                    "type": "tag"
+                  },
+                  {
+                    "params": [
+                      "none"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": false,
+                "measurement": "logins.count",
+                "policy": "default",
+                "query": "SELECT 8 * count(\"value\") FROM \"logins.count\" WHERE $timeFilter GROUP BY time($interval), \"datacenter\" fill(none)",
+                "rawQuery": true,
+                "refId": "B",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [],
+                      "type": "count"
+                    }
+                  ]
+                ],
+                "tags": []
+              },
+              {
+                "groupBy": [
+                  {
+                    "params": [
+                      "$interval"
+                    ],
+                    "type": "time"
+                  },
+                  {
+                    "params": [
+                      "null"
+                    ],
+                    "type": "fill"
+                  }
+                ],
+                "hide": true,
+                "measurement": "cpu",
+                "policy": "default",
+                "refId": "A",
+                "resultFormat": "time_series",
+                "select": [
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [],
+                      "type": "mean"
+                    }
+                  ],
+                  [
+                    {
+                      "params": [
+                        "value"
+                      ],
+                      "type": "field"
+                    },
+                    {
+                      "params": [],
+                      "type": "sum"
+                    }
+                  ]
+                ],
+                "tags": []
+              }
+            ],
+            "thresholds": [
+              {
+                "colorMode": "critical",
+                "fill": true,
+                "line": true,
+                "op": "gt",
+                "value": 10
+              }
+            ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Panel Title",
+            "tooltip": {
+              "msResolution": false,
+              "ordering": "alphabetical",
+              "shared": true,
+              "sort": 0,
+              "value_type": "cumulative"
+            },
+            "type": "graph",
+            "xaxis": {
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "short",
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              },
+              {
+                "format": "short",
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              }
+            ]
+          },
+          {
+            "editable": true,
+            "error": false,
+            "id": 2,
+            "isNew": true,
+            "limit": 10,
+            "links": [],
+            "show": "current",
+            "span": 2,
+            "stateFilter": [
+              "alerting"
+            ],
+            "title": "Alert status",
+            "type": "alertlist"
+          }
+        ],
+        "title": "Row"
+      }
+    ],
+    "time": {
+      "from": "now-5m",
+      "to": "now"
+    },
+    "timepicker": {
+      "now": true,
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d"
+      ]
+    },
+    "templating": {
+      "list": []
+    },
+    "annotations": {
+      "list": []
+    },
+    "schemaVersion": 13,
+    "version": 120,
+    "links": [],
+    "gnetId": null
+  }

pkg/services/alerting/test-data/panel-with-id-0.json

@@ -0,0 +1,63 @@
+{
+  "id": 57,
+  "title": "Graphite 4",
+  "originalTitle": "Graphite 4",
+  "tags": ["graphite"],
+  "rows": [
+  {
+    "panels": [
+    {
+      "title": "Active desktop users",
+      "id": 0,
+      "editable": true,
+      "type": "graph",
+      "targets": [
+      {
+        "refId": "A",
+        "target": "aliasByNode(statsd.fakesite.counters.session_start.desktop.count, 4)"
+      }
+      ],
+      "datasource": null,
+      "alert": {
+        "name": "name1",
+        "message": "desc1",
+        "handler": 1,
+        "frequency": "60s",
+        "conditions": [
+        {
+          "type": "query",
+          "query": {"params": ["A", "5m", "now"]},
+          "reducer": {"type": "avg", "params": []},
+          "evaluator": {"type": ">", "params": [100]}
+        }
+        ]
+      }
+    },
+    {
+      "title": "Active mobile users",
+      "id": 4,
+      "targets": [
+        {"refId": "A", "target": ""},
+        {"refId": "B", "target": "aliasByNode(statsd.fakesite.counters.session_start.mobile.count, 4)"}
+      ],
+      "datasource": "graphite2",
+      "alert": {
+        "name": "name2",
+        "message": "desc2",
+        "handler": 0,
+        "frequency": "60s",
+        "severity": "warning",
+        "conditions": [
+        {
+          "type": "query",
+          "query":  {"params": ["B", "5m", "now"]},
+          "reducer": {"type": "avg", "params": []},
+          "evaluator": {"type": ">", "params": [100]}
+        }
+        ]
+      }
+    }
+    ]
+  }
+]
+      }

pkg/services/dashboards/dashboard_service.go

@@ -13,6 +13,7 @@ import (
 // DashboardService service for operating on dashboards
 type DashboardService interface {
 	SaveDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error)
+	ImportDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error)
 }
 
 // DashboardProvisioningService service for operating on provisioned dashboards
@@ -214,6 +215,20 @@ func (dr *dashboardServiceImpl) SaveDashboard(dto *SaveDashboardDTO) (*models.Da
 	return cmd.Result, nil
 }
 
+func (dr *dashboardServiceImpl) ImportDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error) {
+	cmd, err := dr.buildSaveDashboardCommand(dto, false)
+	if err != nil {
+		return nil, err
+	}
+
+	err = bus.Dispatch(cmd)
+	if err != nil {
+		return nil, err
+	}
+
+	return cmd.Result, nil
+}
+
 type FakeDashboardService struct {
 	SaveDashboardResult *models.Dashboard
 	SaveDashboardError  error
@@ -230,6 +245,10 @@ func (s *FakeDashboardService) SaveDashboard(dto *SaveDashboardDTO) (*models.Das
 	return s.SaveDashboardResult, s.SaveDashboardError
 }
 
+func (s *FakeDashboardService) ImportDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error) {
+	return s.SaveDashboard(dto)
+}
+
 func MockDashboardService(mock *FakeDashboardService) {
 	NewService = func() DashboardService {
 		return mock

pkg/services/sqlstore/alert.go

@@ -252,7 +252,7 @@ func SetAlertState(cmd *m.SetAlertStateCommand) error {
 		}
 
 		alert.State = cmd.State
-		alert.StateChanges += 1
+		alert.StateChanges++
 		alert.NewStateDate = time.Now()
 		alert.EvalData = cmd.EvalData
 

pkg/services/sqlstore/dashboard_folder_test.go

@@ -49,6 +49,7 @@ func TestDashboardFolderDataAccess(t *testing.T) {
 						OrgId:        1, DashboardIds: []int64{folder.Id, dashInRoot.Id},
 					}
 					err := SearchDashboards(query)
+
 					So(err, ShouldBeNil)
 					So(len(query.Result), ShouldEqual, 1)
 					So(query.Result[0].Id, ShouldEqual, dashInRoot.Id)

pkg/services/sqlstore/migrator/migrator.go

@@ -125,7 +125,7 @@ func (mg *Migrator) exec(m Migration, sess *xorm.Session) error {
 	condition := m.GetCondition()
 	if condition != nil {
 		sql, args := condition.Sql(mg.dialect)
-		results, err := sess.Query(sql, args...)
+		results, err := sess.SQL(sql).Query(args...)
 		if err != nil || len(results) == 0 {
 			mg.Logger.Info("Skipping migration condition not fulfilled", "id", m.Id())
 			return sess.Rollback()

pkg/services/sqlstore/org_test.go

@@ -2,6 +2,7 @@ package sqlstore
 
 import (
 	"testing"
+	"time"
 
 	. "github.com/smartystreets/goconvey/convey"
 
@@ -241,6 +242,8 @@ func TestAccountDataAccess(t *testing.T) {
 func testHelperUpdateDashboardAcl(dashboardId int64, items ...m.DashboardAcl) error {
 	cmd := m.UpdateDashboardAclCommand{DashboardId: dashboardId}
 	for _, item := range items {
+		item.Created = time.Now()
+		item.Updated = time.Now()
 		cmd.Items = append(cmd.Items, &item)
 	}
 	return UpdateDashboardAcl(&cmd)

pkg/services/sqlstore/quota.go

@@ -2,6 +2,7 @@ package sqlstore
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	m "github.com/grafana/grafana/pkg/models"
@@ -98,8 +99,9 @@ func UpdateOrgQuota(cmd *m.UpdateOrgQuotaCmd) error {
 	return inTransaction(func(sess *DBSession) error {
 		//Check if quota is already defined in the DB
 		quota := m.Quota{
-			Target: cmd.Target,
+			Target:  cmd.Target,
-			OrgId:  cmd.OrgId,
+			OrgId:   cmd.OrgId,
+			Updated: time.Now(),
 		}
 		has, err := sess.Get(&quota)
 		if err != nil {
@@ -107,6 +109,7 @@ func UpdateOrgQuota(cmd *m.UpdateOrgQuotaCmd) error {
 		}
 		quota.Limit = cmd.Limit
 		if has == false {
+			quota.Created = time.Now()
 			//No quota in the DB for this target, so create a new one.
 			if _, err := sess.Insert(&quota); err != nil {
 				return err
@@ -198,8 +201,9 @@ func UpdateUserQuota(cmd *m.UpdateUserQuotaCmd) error {
 	return inTransaction(func(sess *DBSession) error {
 		//Check if quota is already defined in the DB
 		quota := m.Quota{
-			Target: cmd.Target,
+			Target:  cmd.Target,
-			UserId: cmd.UserId,
+			UserId:  cmd.UserId,
+			Updated: time.Now(),
 		}
 		has, err := sess.Get(&quota)
 		if err != nil {
@@ -207,6 +211,7 @@ func UpdateUserQuota(cmd *m.UpdateUserQuotaCmd) error {
 		}
 		quota.Limit = cmd.Limit
 		if has == false {
+			quota.Created = time.Now()
 			//No quota in the DB for this target, so create a new one.
 			if _, err := sess.Insert(&quota); err != nil {
 				return err

pkg/services/sqlstore/quota_test.go

@@ -104,12 +104,12 @@ func TestQuotaCommandsAndQueries(t *testing.T) {
 			})
 		})
 		Convey("Given saved user quota for org", func() {
-			userQoutaCmd := m.UpdateUserQuotaCmd{
+			userQuotaCmd := m.UpdateUserQuotaCmd{
 				UserId: userId,
 				Target: "org_user",
 				Limit:  10,
 			}
-			err := UpdateUserQuota(&userQoutaCmd)
+			err := UpdateUserQuota(&userQuotaCmd)
 			So(err, ShouldBeNil)
 
 			Convey("Should be able to get saved quota by user id and target", func() {

pkg/services/sqlstore/sqlstore.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
@@ -33,6 +34,7 @@ type DatabaseConfig struct {
 	ServerCertName                             string
 	MaxOpenConn                                int
 	MaxIdleConn                                int
+	ConnMaxLifetime                            int
 }
 
 var (
@@ -157,18 +159,20 @@ func getEngine() (*xorm.Engine, error) {
 	engine, err := xorm.NewEngine(DbCfg.Type, cnnstr)
 	if err != nil {
 		return nil, err
-	} else {
-		engine.SetMaxOpenConns(DbCfg.MaxOpenConn)
-		engine.SetMaxIdleConns(DbCfg.MaxIdleConn)
-		debugSql := setting.Cfg.Section("database").Key("log_queries").MustBool(false)
-		if !debugSql {
-			engine.SetLogger(&xorm.DiscardLogger{})
-		} else {
-			engine.SetLogger(NewXormLogger(log.LvlInfo, log.New("sqlstore.xorm")))
-			engine.ShowSQL(true)
-			engine.ShowExecTime(true)
-		}
 	}
+
+	engine.SetMaxOpenConns(DbCfg.MaxOpenConn)
+	engine.SetMaxIdleConns(DbCfg.MaxIdleConn)
+	engine.SetConnMaxLifetime(time.Second * time.Duration(DbCfg.ConnMaxLifetime))
+	debugSql := setting.Cfg.Section("database").Key("log_queries").MustBool(false)
+	if !debugSql {
+		engine.SetLogger(&xorm.DiscardLogger{})
+	} else {
+		engine.SetLogger(NewXormLogger(log.LvlInfo, log.New("sqlstore.xorm")))
+		engine.ShowSQL(true)
+		engine.ShowExecTime(true)
+	}
+
 	return engine, nil
 }
 
@@ -202,6 +206,7 @@ func LoadConfig() {
 	}
 	DbCfg.MaxOpenConn = sec.Key("max_open_conn").MustInt(0)
 	DbCfg.MaxIdleConn = sec.Key("max_idle_conn").MustInt(0)
+	DbCfg.ConnMaxLifetime = sec.Key("conn_max_lifetime").MustInt(14400)
 
 	if DbCfg.Type == "sqlite3" {
 		UseSQLite3 = true
@@ -225,8 +230,8 @@ var (
 
 func InitTestDB(t *testing.T) *xorm.Engine {
 	selectedDb := dbSqlite
-	//selectedDb := dbMySql
+	// selectedDb := dbMySql
-	//selectedDb := dbPostgres
+	// selectedDb := dbPostgres
 
 	var x *xorm.Engine
 	var err error
@@ -245,6 +250,9 @@ func InitTestDB(t *testing.T) *xorm.Engine {
 		x, err = xorm.NewEngine(sqlutil.TestDB_Sqlite3.DriverName, sqlutil.TestDB_Sqlite3.ConnStr)
 	}
 
+	x.DatabaseTZ = time.UTC
+	x.TZLocation = time.UTC
+
 	// x.ShowSQL()
 
 	if err != nil {

pkg/setting/setting.go

@@ -131,7 +131,8 @@ var (
 	PluginAppsSkipVerifyTLS bool
 
 	// Session settings.
-	SessionOptions session.Options
+	SessionOptions         session.Options
+	SessionConnMaxLifetime int64
 
 	// Global setting objects.
 	Cfg          *ini.File
@@ -634,6 +635,8 @@ func readSessionConfig() {
 	if SessionOptions.CookiePath == "" {
 		SessionOptions.CookiePath = "/"
 	}
+
+	SessionConnMaxLifetime = Cfg.Section("session").Key("conn_max_lifetime").MustInt64(14400)
 }
 
 func initLogging() {

pkg/social/generic_oauth.go

@@ -180,6 +180,7 @@ type UserInfoJson struct {
 
 func (s *SocialGenericOAuth) UserInfo(client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) {
 	var data UserInfoJson
+	var err error
 
 	if s.extractToken(&data, token) != true {
 		response, err := HttpGet(client, s.apiUrl)
@@ -193,20 +194,17 @@ func (s *SocialGenericOAuth) UserInfo(client *http.Client, token *oauth2.Token)
 		}
 	}
 
-	name, err := s.extractName(data)
+	name := s.extractName(&data)
-	if err != nil {
+
-		return nil, err
+	email := s.extractEmail(&data)
+	if email == "" {
+		email, err = s.FetchPrivateEmail(client)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	email, err := s.extractEmail(data, client)
+	login := s.extractLogin(&data, email)
-	if err != nil {
-		return nil, err
-	}
-
-	login, err := s.extractLogin(data, email)
-	if err != nil {
-		return nil, err
-	}
 
 	userInfo := &BasicUserInfo{
 		Name:  name,
@@ -251,49 +249,55 @@ func (s *SocialGenericOAuth) extractToken(data *UserInfoJson, token *oauth2.Toke
 		return false
 	}
 
+	email := s.extractEmail(data)
+	if email == "" {
+		s.log.Debug("No email found in id_token", "json", string(payload), "data", data)
+		return false
+	}
+
 	s.log.Debug("Received id_token", "json", string(payload), "data", data)
 	return true
 }
 
-func (s *SocialGenericOAuth) extractEmail(data UserInfoJson, client *http.Client) (string, error) {
+func (s *SocialGenericOAuth) extractEmail(data *UserInfoJson) string {
 	if data.Email != "" {
-		return data.Email, nil
+		return data.Email
 	}
 
 	if data.Attributes["email:primary"] != nil {
-		return data.Attributes["email:primary"][0], nil
+		return data.Attributes["email:primary"][0]
 	}
 
 	if data.Upn != "" {
 		emailAddr, emailErr := mail.ParseAddress(data.Upn)
 		if emailErr == nil {
-			return emailAddr.Address, nil
+			return emailAddr.Address
 		}
 	}
 
-	return s.FetchPrivateEmail(client)
+	return ""
 }
 
-func (s *SocialGenericOAuth) extractLogin(data UserInfoJson, email string) (string, error) {
+func (s *SocialGenericOAuth) extractLogin(data *UserInfoJson, email string) string {
 	if data.Login != "" {
-		return data.Login, nil
+		return data.Login
 	}
 
 	if data.Username != "" {
-		return data.Username, nil
+		return data.Username
 	}
 
-	return email, nil
+	return email
 }
 
-func (s *SocialGenericOAuth) extractName(data UserInfoJson) (string, error) {
+func (s *SocialGenericOAuth) extractName(data *UserInfoJson) string {
 	if data.Name != "" {
-		return data.Name, nil
+		return data.Name
 	}
 
 	if data.DisplayName != "" {
-		return data.DisplayName, nil
+		return data.DisplayName
 	}
 
-	return "", nil
+	return ""
 }

pkg/tsdb/postgres/postgres.go

@@ -53,7 +53,8 @@ func generateConnectionString(datasource *models.DataSource) string {
 	}
 
 	sslmode := datasource.JsonData.Get("sslmode").MustString("verify-full")
-	return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", url.PathEscape(datasource.User), url.PathEscape(password), url.PathEscape(datasource.Url), url.PathEscape(datasource.Database), url.QueryEscape(sslmode))
+	u := &url.URL{Scheme: "postgres", User: url.UserPassword(datasource.User, password), Host: datasource.Url, Path: datasource.Database, RawQuery: "sslmode=" + sslmode}
+	return u.String()
 }
 
 func (e *PostgresQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {

public/app/core/components/scroll/scroll.ts

@@ -8,6 +8,7 @@ export function geminiScrollbar() {
     link: function(scope, elem, attrs) {
       let scrollbar = new PerfectScrollbar(elem[0], {
         wheelPropagation: true,
+        wheelSpeed: 3,
       });
       let lastPos = 0;
 

public/app/core/components/sidemenu/sidemenu.html

@@ -1,73 +1,78 @@
 <a class="sidemenu__logo" ng-click="ctrl.toggleSideMenu()">
-	<img src="public/img/grafana_icon.svg"></img>
+  <img src="public/img/grafana_icon.svg"></img>
 </a>
 
 <a class="sidemenu__logo_small_breakpoint" ng-click="ctrl.toggleSideMenuSmallBreakpoint()">
   <i class="fa fa-bars"></i>
-  <span class="sidemenu__close"><i class="fa fa-times"></i>&nbsp;Close</span>
+  <span class="sidemenu__close">
+    <i class="fa fa-times"></i>&nbsp;Close</span>
 </a>
 
 <div class="sidemenu__top">
-	<div ng-repeat="item in ::ctrl.mainLinks" class="sidemenu-item dropdown">
+  <div ng-repeat="item in ::ctrl.mainLinks" class="sidemenu-item dropdown">
-		<a href="{{::item.url}}" class="sidemenu-link" target="{{::item.target}}">
+    <a href="{{::item.url}}" class="sidemenu-link" target="{{::item.target}}">
-			<span class="icon-circle sidemenu-icon">
+      <span class="icon-circle sidemenu-icon">
-				<i class="{{::item.icon}}" ng-show="::item.icon"></i>
+        <i class="{{::item.icon}}" ng-show="::item.icon"></i>
-				<img ng-src="{{::item.img}}" ng-show="::item.img">
+        <img ng-src="{{::item.img}}" ng-show="::item.img">
-			</span>
+      </span>
-		</a>
+    </a>
-		<ul class="dropdown-menu dropdown-menu--sidemenu" role="menu" ng-if="::item.children">
+    <ul class="dropdown-menu dropdown-menu--sidemenu" role="menu" ng-if="::item.children">
-			<li class="side-menu-header">
+      <li class="side-menu-header">
-				<span class="sidemenu-item-text">{{::item.text}}</span>
+        <span class="sidemenu-item-text">{{::item.text}}</span>
-			</li>
+      </li>
-			<li ng-repeat="child in ::item.children" ng-class="{divider: child.divider}">
+      <li ng-repeat="child in ::item.children" ng-class="{divider: child.divider}">
-				<a href="{{::child.url}}">
+        <a href="{{::child.url}}">
-					<i class="{{::child.icon}}" ng-show="::child.icon"></i>
+          <i class="{{::child.icon}}" ng-show="::child.icon"></i>
-					{{::child.text}}
+          {{::child.text}}
-				</a>
+        </a>
-			</li>
+      </li>
-		</ul>
+    </ul>
-	</div>
+  </div>
 </div>
 
 <div class="sidemenu__bottom">
-	<div ng-show="::!ctrl.isSignedIn" class="sidemenu-item">
+  <div ng-show="::!ctrl.isSignedIn" class="sidemenu-item">
-		<a href="{{ctrl.loginUrl}}" class="sidemenu-link" target="_self">
+    <a href="{{ctrl.loginUrl}}" class="sidemenu-link" target="_self">
-			<span class="icon-circle sidemenu-icon"><i class="fa fa-fw fa-sign-in"></i></span>
+      <span class="icon-circle sidemenu-icon">
-		</a>
+        <i class="fa fa-fw fa-sign-in"></i>
-		<ul class="dropdown-menu dropdown-menu--sidemenu" role="menu">
+      </span>
-			<li class="side-menu-header">
+    </a>
-				<span class="sidemenu-item-text">Sign In</span>
+    <a href="{{ctrl.loginUrl}}">
-			</li>
+      <ul class="dropdown-menu dropdown-menu--sidemenu" role="menu">
-		</ul>
+        <li class="side-menu-header">
-	</div>
+          <span class="sidemenu-item-text">Sign In</span>
+        </li>
+      </ul>
+    </a>
+  </div>
 
-	<div ng-repeat="item in ::ctrl.bottomNav" class="sidemenu-item dropdown dropup">
+  <div ng-repeat="item in ::ctrl.bottomNav" class="sidemenu-item dropdown dropup">
-		<a href="{{::item.url}}" class="sidemenu-link" target="{{::item.target}}">
+    <a href="{{::item.url}}" class="sidemenu-link" target="{{::item.target}}">
-			<span class="icon-circle sidemenu-icon">
+      <span class="icon-circle sidemenu-icon">
-				<i class="{{::item.icon}}" ng-show="::item.icon"></i>
+        <i class="{{::item.icon}}" ng-show="::item.icon"></i>
-				<img ng-src="{{::item.img}}" ng-show="::item.img">
+        <img ng-src="{{::item.img}}" ng-show="::item.img">
-			</span>
+      </span>
-		</a>
+    </a>
-		<ul class="dropdown-menu dropdown-menu--sidemenu" role="menu">
+    <ul class="dropdown-menu dropdown-menu--sidemenu" role="menu">
-			<li ng-if="item.showOrgSwitcher" class="sidemenu-org-switcher">
+      <li ng-if="item.showOrgSwitcher" class="sidemenu-org-switcher">
-				<a ng-click="ctrl.switchOrg()">
+        <a ng-click="ctrl.switchOrg()">
-					<div>
+          <div>
-						<div class="sidemenu-org-switcher__org-name">{{ctrl.contextSrv.user.orgName}}</div>
+            <div class="sidemenu-org-switcher__org-name">{{ctrl.contextSrv.user.orgName}}</div>
-						<div class="sidemenu-org-switcher__org-current">Current Org:</div>
+            <div class="sidemenu-org-switcher__org-current">Current Org:</div>
-					</div>
+          </div>
-					<div class="sidemenu-org-switcher__switch"><i class="fa fa-fw fa-random"></i>Switch</div>
+          <div class="sidemenu-org-switcher__switch">
-				</a>
+            <i class="fa fa-fw fa-random"></i>Switch</div>
-			</li>
+        </a>
-			<li ng-repeat="child in ::item.children" ng-class="{divider: child.divider}" ng-hide="::child.hideFromMenu">
+      </li>
-				<a href="{{::child.url}}" target="{{::child.target}}" ng-click="ctrl.itemClicked(child, $event)">
+      <li ng-repeat="child in ::item.children" ng-class="{divider: child.divider}" ng-hide="::child.hideFromMenu">
-					<i class="{{::child.icon}}" ng-show="::child.icon"></i>
+        <a href="{{::child.url}}" target="{{::child.target}}" ng-click="ctrl.itemClicked(child, $event)">
-					{{::child.text}}
+          <i class="{{::child.icon}}" ng-show="::child.icon"></i>
-				</a>
+          {{::child.text}}
-			</li>
+        </a>
-			<li class="side-menu-header">
+      </li>
-				<span class="sidemenu-item-text">{{::item.text}}</span>
+      <li class="side-menu-header">
-			</li>
+        <span class="sidemenu-item-text">{{::item.text}}</span>
-		</ul>
+      </li>
-	</div>
+    </ul>
+  </div>
 </div>
-

public/app/features/alerting/notification_edit_ctrl.ts

@@ -43,6 +43,7 @@ export class AlertNotificationEditCtrl {
         return this.backendSrv.get(`/api/alert-notifications/${this.$routeParams.id}`).then(result => {
           this.navModel.breadcrumbs.push({ text: result.name });
           this.navModel.node = { text: result.name };
+          result.settings = _.defaults(result.settings, this.defaults.settings);
           return result;
         });
       })
@@ -89,7 +90,7 @@ export class AlertNotificationEditCtrl {
   }
 
   typeChanged() {
-    this.model.settings = {};
+    this.model.settings = _.defaults({}, this.defaults.settings);
     this.notifierTemplateId = this.getNotifierTemplateId(this.model.type);
   }
 

public/app/features/dashboard/dashgrid/DashboardRow.tsx

@@ -4,6 +4,7 @@ import { PanelModel } from '../panel_model';
 import { PanelContainer } from './PanelContainer';
 import templateSrv from 'app/features/templating/template_srv';
 import appEvents from 'app/core/app_events';
+import config from 'app/core/config';
 
 export interface DashboardRowProps {
   panel: PanelModel;
@@ -94,14 +95,16 @@ export class DashboardRow extends React.Component<DashboardRowProps, any> {
           {title}
           <span className="dashboard-row__panel_count">({hiddenPanels} hidden panels)</span>
         </a>
-        <div className="dashboard-row__actions">
+        {config.bootData.user.orgRole !== 'Viewer' && (
-          <a className="pointer" onClick={this.openSettings}>
+          <div className="dashboard-row__actions">
-            <i className="fa fa-cog" />
+            <a className="pointer" onClick={this.openSettings}>
-          </a>
+              <i className="fa fa-cog" />
-          <a className="pointer" onClick={this.delete}>
+            </a>
-            <i className="fa fa-trash" />
+            <a className="pointer" onClick={this.delete}>
-          </a>
+              <i className="fa fa-trash" />
-        </div>
+            </a>
+          </div>
+        )}
         <div className="dashboard-row__drag grid-drag-handle" />
       </div>
     );

public/app/features/dashboard/specs/DashboardRow.jest.tsx

@@ -2,19 +2,26 @@ import React from 'react';
 import { shallow } from 'enzyme';
 import { DashboardRow } from '../dashgrid/DashboardRow';
 import { PanelModel } from '../panel_model';
+import config from '../../../core/config';
 
 describe('DashboardRow', () => {
   let wrapper, panel, getPanelContainer, dashboardMock;
 
   beforeEach(() => {
-    dashboardMock = {toggleRow: jest.fn()};
+    dashboardMock = { toggleRow: jest.fn() };
+
+    config.bootData = {
+      user: {
+        orgRole: 'Admin',
+      },
+    };
 
     getPanelContainer = jest.fn().mockReturnValue({
       getDashboard: jest.fn().mockReturnValue(dashboardMock),
-      getPanelLoader: jest.fn()
+      getPanelLoader: jest.fn(),
     });
 
-    panel = new PanelModel({collapsed: false});
+    panel = new PanelModel({ collapsed: false });
     wrapper = shallow(<DashboardRow panel={panel} getPanelContainer={getPanelContainer} />);
   });
 
@@ -30,4 +37,14 @@ describe('DashboardRow', () => {
     expect(dashboardMock.toggleRow.mock.calls).toHaveLength(1);
   });
 
+  it('should have two actions as admin', () => {
+    expect(wrapper.find('.dashboard-row__actions .pointer')).toHaveLength(2);
+  });
+
+  it('should have zero actions as viewer', () => {
+    config.bootData.user.orgRole = 'Viewer';
+    panel = new PanelModel({ collapsed: false });
+    wrapper = shallow(<DashboardRow panel={panel} getPanelContainer={getPanelContainer} />);
+    expect(wrapper.find('.dashboard-row__actions .pointer')).toHaveLength(0);
+  });
 });

public/app/features/dashboard/specs/unsaved_changes_srv_specs.ts

@@ -66,6 +66,11 @@ describe('unsavedChangesSrv', function() {
     expect(tracker.hasChanges()).to.be(false);
   });
 
+  it('Should ignore .iteration changes', () => {
+    dash.iteration = new Date().getTime() + 1;
+    expect(tracker.hasChanges()).to.be(false);
+  });
+
   it.skip('Should ignore row collapse change', function() {
     dash.rows[0].collapse = true;
     expect(tracker.hasChanges()).to.be(false);

public/app/features/dashboard/specs/viewstate_srv_specs.ts

@@ -30,7 +30,10 @@ describe('when updating view state', function() {
   beforeEach(
     angularMocks.inject(function(dashboardViewStateSrv, $location, $rootScope) {
       $rootScope.onAppEvent = function() {};
-      $rootScope.dashboard = { meta: {} };
+      $rootScope.dashboard = {
+        meta: {},
+        panels: [],
+      };
       viewState = dashboardViewStateSrv.create($rootScope);
       location = $location;
     })

public/app/features/dashboard/unsaved_changes_srv.ts

@@ -97,6 +97,9 @@ export class Tracker {
     dash.refresh = 0;
     dash.schemaVersion = 0;
 
+    // ignore iteration property
+    delete dash.iteration;
+
     // filter row and panels properties that should be ignored
     dash.rows = _.filter(dash.rows, function(row) {
       if (row.repeatRowId) {

public/app/features/dashboard/view_state_srv.ts

@@ -1,6 +1,7 @@
 import angular from 'angular';
 import _ from 'lodash';
 import config from 'app/core/config';
+import { DashboardModel } from './dashboard_model';
 
 // represents the transient view state
 // like fullscreen panel & edit
@@ -8,7 +9,7 @@ export class DashboardViewState {
   state: any;
   panelScopes: any;
   $scope: any;
-  dashboard: any;
+  dashboard: DashboardModel;
   editStateChanged: any;
   fullscreenPanel: any;
   oldTimeRange: any;
@@ -89,6 +90,12 @@ export class DashboardViewState {
       }
     }
 
+    if ((this.state.fullscreen || this.dashboard.meta.soloMode) && this.state.panelId) {
+      // Trying to render panel in fullscreen when it's in the collapsed row causes an issue.
+      // So in this case expand collapsed row first.
+      this.toggleCollapsedPanelRow(this.state.panelId);
+    }
+
     // if no edit state cleanup tab parm
     if (!this.state.edit) {
       delete this.state.tab;
@@ -103,6 +110,19 @@ export class DashboardViewState {
     this.syncState();
   }
 
+  toggleCollapsedPanelRow(panelId) {
+    for (let panel of this.dashboard.panels) {
+      if (panel.collapsed) {
+        for (let rowPanel of panel.panels) {
+          if (rowPanel.id === panelId) {
+            this.dashboard.toggleRow(panel);
+            return;
+          }
+        }
+      }
+    }
+  }
+
   syncState() {
     if (this.panelScopes.length === 0) {
       return;

public/app/features/panel/metrics_panel_ctrl.ts

@@ -78,8 +78,11 @@ class MetricsPanelCtrl extends PanelCtrl {
         data = data.data;
       }
 
-      this.events.emit('data-snapshot-load', data);
+      // Defer panel rendering till the next digest cycle.
-      return;
+      // For some reason snapshot panels don't init at this time, so this helps to avoid rendering issues.
+      return this.$timeout(() => {
+        this.events.emit('data-snapshot-load', data);
+      });
     }
 
     // // ignore if we have data stream

public/app/plugins/datasource/prometheus/dashboards/prometheus_2_stats.json

@@ -348,7 +348,7 @@
           "tableColumn": "",
           "targets": [
             {
-              "expr": "tsdb_wal_corruptions_total{job=\"prometheus\"}",
+              "expr": "prometheus_tsdb_wal_corruptions_total{job=\"prometheus\"}",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "",
@@ -1048,7 +1048,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "max(prometheus_evaluator_duration_seconds{job=\"prometheus\", quantile!=\"0.01\", quantile!=\"0.05\"}) by (quantile)",
+              "expr": "max(prometheus_rule_group_duration_seconds{job=\"prometheus\"}) by (quantile)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 2,
@@ -1060,7 +1060,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Rule Eval Duration",
+          "title": "Rule Group Eval Duration",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -1124,7 +1124,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(prometheus_evaluator_iterations_missed_total{job=\"prometheus\"}[5m])",
+              "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\"}[5m])",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "missed",
@@ -1132,15 +1132,7 @@
               "step": 10
             },
             {
-              "expr": "rate(prometheus_evaluator_iterations_skipped_total{job=\"prometheus\"}[5m])",
+              "expr": "rate(prometheus_rule_group_iterations_total{job=\"prometheus\"}[5m])",
-              "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "skipped",
-              "refId": "C",
-              "step": 10
-            },
-            {
-              "expr": "rate(prometheus_evaluator_iterations_total{job=\"prometheus\"}[5m])",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "iterations",
@@ -1151,7 +1143,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Rule Eval Activity",
+          "title": "Rule Group Eval Activity",
           "tooltip": {
             "shared": true,
             "sort": 0,

public/app/routes/routes.ts

@@ -22,6 +22,12 @@ export function setupAngularRoutes($routeProvider, $locationProvider) {
       reloadOnSearch: false,
       pageClass: 'page-dashboard',
     })
+    .when('/d/:uid', {
+      templateUrl: 'public/app/partials/dashboard.html',
+      controller: 'LoadDashboardCtrl',
+      reloadOnSearch: false,
+      pageClass: 'page-dashboard',
+    })
     .when('/dashboard/:type/:slug', {
       templateUrl: 'public/app/partials/dashboard.html',
       controller: 'LoadDashboardCtrl',
@@ -98,6 +104,11 @@ export function setupAngularRoutes($routeProvider, $locationProvider) {
       controller: 'FolderDashboardsCtrl',
       controllerAs: 'ctrl',
     })
+    .when('/dashboards/f/:uid', {
+      templateUrl: 'public/app/features/dashboard/partials/folder_dashboards.html',
+      controller: 'FolderDashboardsCtrl',
+      controllerAs: 'ctrl',
+    })
     .when('/org', {
       templateUrl: 'public/app/features/org/partials/orgDetails.html',
       controller: 'OrgDetailsCtrl',

public/img/resize-handle-white.svg

@@ -0,0 +1,11 @@
+<?xml version="1.0" standalone="no"?>
+<!-- Generator: Adobe Fireworks CS6, Export SVG Extension by Aaron Beall (http://fireworks.abeall.com) . Version: 0.6.1  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg id="Untitled-Page%201" viewBox="0 0 6 6" style="background-color:#ffffff00" version="1.1"
+	xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve"
+	x="0px" y="0px" width="6px" height="6px"
+>
+	<g opacity="0.302">
+		<path d="M 6 6 L 0 6 L 0 4.2 L 4 4.2 L 4.2 4.2 L 4.2 0 L 6 0 L 6 6 L 6 6 Z" fill="#FFFFFF"/>
+	</g>
+</svg>

public/sass/_variables.dark.scss

@@ -259,6 +259,7 @@ $navbar-button-border: #2f2f32;
 // Sidemenu
 // -------------------------
 $side-menu-bg: $black;
+$side-menu-bg-mobile: $side-menu-bg;
 $side-menu-item-hover-bg: $dark-2;
 $side-menu-shadow: 0 0 20px black;
 $side-menu-link-color: $link-color;

public/sass/_variables.light.scss

@@ -200,6 +200,7 @@ $input-invalid-border-color: lighten($red, 5%);
 // Sidemenu
 // -------------------------
 $side-menu-bg: $dark-2;
+$side-menu-bg-mobile: rgba(0, 0, 0, 0); //$gray-6;
 $side-menu-item-hover-bg: $gray-1;
 $side-menu-shadow: 5px 0px 10px -5px $gray-1;
 $side-menu-link-color: $gray-6;

public/sass/components/_dashboard_grid.scss

@@ -44,6 +44,11 @@
     border-right: 2px solid $gray-1;
     border-bottom: 2px solid $gray-1;
   }
+  // temp fix since we use old commit of grid component
+  // this can be removed when we revert to non fork grid component
+  .react-grid-item > .react-resizable-handle {
+    background-image: url('../img/resize-handle-white.svg');
+  }
 }
 
 .theme-light {

public/sass/components/_sidemenu.scss

@@ -71,7 +71,7 @@
         // important to overlap it otherwise it can be hidden
         // again by the mouse getting outside the hover space
         left: $side-menu-width - 2px;
-        @include animation("dropdown-anim 150ms ease-in-out 100ms forwards");
+        @include animation('dropdown-anim 150ms ease-in-out 100ms forwards');
         z-index: $zindex-sidemenu;
       }
     }
@@ -193,9 +193,13 @@ li.sidemenu-org-switcher {
 
 @include media-breakpoint-down(xs) {
   .sidemenu-open--xs {
+    li {
+      font-size: $font-size-lg;
+    }
+
     .sidemenu {
       width: 100%;
-      background: $side-menu-bg;
+      background: $side-menu-bg-mobile;
       position: initial;
       height: auto;
       box-shadow: $side-menu-shadow;
@@ -214,6 +218,9 @@ li.sidemenu-org-switcher {
     .sidemenu__bottom {
       display: block;
     }
+    .sidemenu-item {
+      border-right: 2px solid transparent;
+    }
   }
 
   .sidemenu {

vendor/github.com/go-xorm/core/column.go

@@ -13,12 +13,13 @@ const (
 	ONLYFROMDB
 )
 
-// database column
+// Column defines database column
 type Column struct {
 	Name            string
 	TableName       string
 	FieldName       string
 	SQLType         SQLType
+	IsJSON          bool
 	Length          int
 	Length2         int
 	Nullable        bool
@@ -37,6 +38,7 @@ type Column struct {
 	SetOptions      map[string]int
 	DisableTimeZone bool
 	TimeZone        *time.Location // column specified time zone
+	Comment         string
 }
 
 func NewColumn(name, fieldName string, sqlType SQLType, len1, len2 int, nullable bool) *Column {
@@ -60,6 +62,7 @@ func NewColumn(name, fieldName string, sqlType SQLType, len1, len2 int, nullable
 		IsVersion:       false,
 		DefaultIsEmpty:  false,
 		EnumOptions:     make(map[string]int),
+		Comment:         "",
 	}
 }
 

vendor/github.com/go-xorm/core/dialect.go

@@ -244,6 +244,9 @@ func (b *Base) CreateTableSql(table *Table, tableName, storeEngine, charset stri
 				sql += col.StringNoPk(b.dialect)
 			}
 			sql = strings.TrimSpace(sql)
+			if b.DriverName() == MYSQL && len(col.Comment) > 0 {
+				sql += " COMMENT '" + col.Comment + "'"
+			}
 			sql += ", "
 		}
 

vendor/github.com/go-xorm/core/rows.go

@@ -196,7 +196,7 @@ func (rs *Rows) ScanMap(dest interface{}) error {
 	newDest := make([]interface{}, len(cols))
 	vvv := vv.Elem()
 
-	for i, _ := range cols {
+	for i := range cols {
 		newDest[i] = ReflectNew(vvv.Type().Elem()).Interface()
 		//v := reflect.New(vvv.Type().Elem())
 		//newDest[i] = v.Interface()
@@ -247,6 +247,18 @@ type Row struct {
 	err error // deferred error for easy chaining
 }
 
+// ErrorRow return an error row
+func ErrorRow(err error) *Row {
+	return &Row{
+		err: err,
+	}
+}
+
+// NewRow from rows
+func NewRow(rows *Rows, err error) *Row {
+	return &Row{rows, err}
+}
+
 func (row *Row) Columns() ([]string, error) {
 	if row.err != nil {
 		return nil, row.err

vendor/github.com/go-xorm/core/table.go

@@ -22,6 +22,7 @@ type Table struct {
 	Cacher        Cacher
 	StoreEngine   string
 	Charset       string
+	Comment       string
 }
 
 func (table *Table) Columns() []*Column {

vendor/github.com/go-xorm/core/type.go

@@ -100,7 +100,8 @@ var (
 	LongBlob   = "LONGBLOB"
 	Bytea      = "BYTEA"
 
-	Bool = "BOOL"
+	Bool    = "BOOL"
+	Boolean = "BOOLEAN"
 
 	Serial    = "SERIAL"
 	BigSerial = "BIGSERIAL"
@@ -163,7 +164,7 @@ var (
 	uintTypes = sort.StringSlice{"*uint", "*uint16", "*uint32", "*uint8"}
 )
 
-// !nashtsai! treat following var as interal const values, these are used for reflect.TypeOf comparision
+// !nashtsai! treat following var as interal const values, these are used for reflect.TypeOf comparison
 var (
 	c_EMPTY_STRING       string
 	c_BOOL_DEFAULT       bool

vendor/github.com/go-xorm/xorm/cache_lru.go

@@ -15,13 +15,12 @@ import (
 
 // LRUCacher implments cache object facilities
 type LRUCacher struct {
-	idList   *list.List
+	idList         *list.List
-	sqlList  *list.List
+	sqlList        *list.List
-	idIndex  map[string]map[string]*list.Element
+	idIndex        map[string]map[string]*list.Element
-	sqlIndex map[string]map[string]*list.Element
+	sqlIndex       map[string]map[string]*list.Element
-	store    core.CacheStore
+	store          core.CacheStore
-	mutex    sync.Mutex
+	mutex          sync.Mutex
-	// maxSize    int
 	MaxElementSize int
 	Expired        time.Duration
 	GcInterval     time.Duration
@@ -54,8 +53,6 @@ func (m *LRUCacher) RunGC() {
 
 // GC check ids lit and sql list to remove all element expired
 func (m *LRUCacher) GC() {
-	//fmt.Println("begin gc ...")
-	//defer fmt.Println("end gc ...")
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 	var removedNum int
@@ -64,12 +61,10 @@ func (m *LRUCacher) GC() {
 			time.Now().Sub(e.Value.(*idNode).lastVisit) > m.Expired {
 			removedNum++
 			next := e.Next()
-			//fmt.Println("removing ...", e.Value)
 			node := e.Value.(*idNode)
 			m.delBean(node.tbName, node.id)
 			e = next
 		} else {
-			//fmt.Printf("removing %d cache nodes ..., left %d\n", removedNum, m.idList.Len())
 			break
 		}
 	}
@@ -80,12 +75,10 @@ func (m *LRUCacher) GC() {
 			time.Now().Sub(e.Value.(*sqlNode).lastVisit) > m.Expired {
 			removedNum++
 			next := e.Next()
-			//fmt.Println("removing ...", e.Value)
 			node := e.Value.(*sqlNode)
 			m.delIds(node.tbName, node.sql)
 			e = next
 		} else {
-			//fmt.Printf("removing %d cache nodes ..., left %d\n", removedNum, m.sqlList.Len())
 			break
 		}
 	}
@@ -116,7 +109,6 @@ func (m *LRUCacher) GetIds(tableName, sql string) interface{} {
 	}
 
 	m.delIds(tableName, sql)
-
 	return nil
 }
 
@@ -134,7 +126,6 @@ func (m *LRUCacher) GetBean(tableName string, id string) interface{} {
 			// if expired, remove the node and return nil
 			if time.Now().Sub(lastTime) > m.Expired {
 				m.delBean(tableName, id)
-				//m.clearIds(tableName)
 				return nil
 			}
 			m.idList.MoveToBack(el)
@@ -148,7 +139,6 @@ func (m *LRUCacher) GetBean(tableName string, id string) interface{} {
 
 	// store bean is not exist, then remove memory's index
 	m.delBean(tableName, id)
-	//m.clearIds(tableName)
 	return nil
 }
 
@@ -166,8 +156,8 @@ func (m *LRUCacher) clearIds(tableName string) {
 // ClearIds clears all sql-ids mapping on table tableName from cache
 func (m *LRUCacher) ClearIds(tableName string) {
 	m.mutex.Lock()
-	defer m.mutex.Unlock()
 	m.clearIds(tableName)
+	m.mutex.Unlock()
 }
 
 func (m *LRUCacher) clearBeans(tableName string) {
@@ -184,14 +174,13 @@ func (m *LRUCacher) clearBeans(tableName string) {
 // ClearBeans clears all beans in some table
 func (m *LRUCacher) ClearBeans(tableName string) {
 	m.mutex.Lock()
-	defer m.mutex.Unlock()
 	m.clearBeans(tableName)
+	m.mutex.Unlock()
 }
 
 // PutIds pus ids into table
 func (m *LRUCacher) PutIds(tableName, sql string, ids interface{}) {
 	m.mutex.Lock()
-	defer m.mutex.Unlock()
 	if _, ok := m.sqlIndex[tableName]; !ok {
 		m.sqlIndex[tableName] = make(map[string]*list.Element)
 	}
@@ -207,12 +196,12 @@ func (m *LRUCacher) PutIds(tableName, sql string, ids interface{}) {
 		node := e.Value.(*sqlNode)
 		m.delIds(node.tbName, node.sql)
 	}
+	m.mutex.Unlock()
 }
 
 // PutBean puts beans into table
 func (m *LRUCacher) PutBean(tableName string, id string, obj interface{}) {
 	m.mutex.Lock()
-	defer m.mutex.Unlock()
 	var el *list.Element
 	var ok bool
 
@@ -229,6 +218,7 @@ func (m *LRUCacher) PutBean(tableName string, id string, obj interface{}) {
 		node := e.Value.(*idNode)
 		m.delBean(node.tbName, node.id)
 	}
+	m.mutex.Unlock()
 }
 
 func (m *LRUCacher) delIds(tableName, sql string) {
@@ -244,8 +234,8 @@ func (m *LRUCacher) delIds(tableName, sql string) {
 // DelIds deletes ids
 func (m *LRUCacher) DelIds(tableName, sql string) {
 	m.mutex.Lock()
-	defer m.mutex.Unlock()
 	m.delIds(tableName, sql)
+	m.mutex.Unlock()
 }
 
 func (m *LRUCacher) delBean(tableName string, id string) {
@@ -261,8 +251,8 @@ func (m *LRUCacher) delBean(tableName string, id string) {
 // DelBean deletes beans in some table
 func (m *LRUCacher) DelBean(tableName string, id string) {
 	m.mutex.Lock()
-	defer m.mutex.Unlock()
 	m.delBean(tableName, id)
+	m.mutex.Unlock()
 }
 
 type idNode struct {

vendor/github.com/go-xorm/xorm/context.go

@@ -0,0 +1,26 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.8
+
+package xorm
+
+import "context"
+
+// PingContext tests if database is alive
+func (engine *Engine) PingContext(ctx context.Context) error {
+	session := engine.NewSession()
+	defer session.Close()
+	return session.PingContext(ctx)
+}
+
+// PingContext test if database is ok
+func (session *Session) PingContext(ctx context.Context) error {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	session.engine.logger.Infof("PING DATABASE %v", session.engine.DriverName())
+	return session.DB().PingContext(ctx)
+}

vendor/github.com/go-xorm/xorm/convert.go

@@ -209,10 +209,10 @@ func convertAssign(dest, src interface{}) error {
 		if src == nil {
 			dv.Set(reflect.Zero(dv.Type()))
 			return nil
-		} else {
-			dv.Set(reflect.New(dv.Type().Elem()))
-			return convertAssign(dv.Interface(), src)
 		}
+
+		dv.Set(reflect.New(dv.Type().Elem()))
+		return convertAssign(dv.Interface(), src)
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 		s := asString(src)
 		i64, err := strconv.ParseInt(s, 10, dv.Type().Bits())
@@ -247,3 +247,102 @@ func convertAssign(dest, src interface{}) error {
 
 	return fmt.Errorf("unsupported Scan, storing driver.Value type %T into type %T", src, dest)
 }
+
+func asKind(vv reflect.Value, tp reflect.Type) (interface{}, error) {
+	switch tp.Kind() {
+	case reflect.Int64:
+		return vv.Int(), nil
+	case reflect.Int:
+		return int(vv.Int()), nil
+	case reflect.Int32:
+		return int32(vv.Int()), nil
+	case reflect.Int16:
+		return int16(vv.Int()), nil
+	case reflect.Int8:
+		return int8(vv.Int()), nil
+	case reflect.Uint64:
+		return vv.Uint(), nil
+	case reflect.Uint:
+		return uint(vv.Uint()), nil
+	case reflect.Uint32:
+		return uint32(vv.Uint()), nil
+	case reflect.Uint16:
+		return uint16(vv.Uint()), nil
+	case reflect.Uint8:
+		return uint8(vv.Uint()), nil
+	case reflect.String:
+		return vv.String(), nil
+	case reflect.Slice:
+		if tp.Elem().Kind() == reflect.Uint8 {
+			v, err := strconv.ParseInt(string(vv.Interface().([]byte)), 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			return v, nil
+		}
+
+	}
+	return nil, fmt.Errorf("unsupported primary key type: %v, %v", tp, vv)
+}
+
+func convertFloat(v interface{}) (float64, error) {
+	switch v.(type) {
+	case float32:
+		return float64(v.(float32)), nil
+	case float64:
+		return v.(float64), nil
+	case string:
+		i, err := strconv.ParseFloat(v.(string), 64)
+		if err != nil {
+			return 0, err
+		}
+		return i, nil
+	case []byte:
+		i, err := strconv.ParseFloat(string(v.([]byte)), 64)
+		if err != nil {
+			return 0, err
+		}
+		return i, nil
+	}
+	return 0, fmt.Errorf("unsupported type: %v", v)
+}
+
+func convertInt(v interface{}) (int64, error) {
+	switch v.(type) {
+	case int:
+		return int64(v.(int)), nil
+	case int8:
+		return int64(v.(int8)), nil
+	case int16:
+		return int64(v.(int16)), nil
+	case int32:
+		return int64(v.(int32)), nil
+	case int64:
+		return v.(int64), nil
+	case []byte:
+		i, err := strconv.ParseInt(string(v.([]byte)), 10, 64)
+		if err != nil {
+			return 0, err
+		}
+		return i, nil
+	case string:
+		i, err := strconv.ParseInt(v.(string), 10, 64)
+		if err != nil {
+			return 0, err
+		}
+		return i, nil
+	}
+	return 0, fmt.Errorf("unsupported type: %v", v)
+}
+
+func asBool(bs []byte) (bool, error) {
+	if len(bs) == 0 {
+		return false, nil
+	}
+	if bs[0] == 0x00 {
+		return false, nil
+	} else if bs[0] == 0x01 {
+		return true, nil
+	}
+	return strconv.ParseBool(string(bs))
+}

vendor/github.com/go-xorm/xorm/dialect_mssql.go

@@ -215,10 +215,10 @@ func (db *mssql) SqlType(c *core.Column) string {
 	var res string
 	switch t := c.SQLType.Name; t {
 	case core.Bool:
-		res = core.TinyInt
+		res = core.Bit
-		if c.Default == "true" {
+		if strings.EqualFold(c.Default, "true") {
 			c.Default = "1"
-		} else if c.Default == "false" {
+		} else {
 			c.Default = "0"
 		}
 	case core.Serial:
@@ -250,6 +250,9 @@ func (db *mssql) SqlType(c *core.Column) string {
 	case core.Uuid:
 		res = core.Varchar
 		c.Length = 40
+	case core.TinyInt:
+		res = core.TinyInt
+		c.Length = 0
 	default:
 		res = t
 	}
@@ -335,9 +338,15 @@ func (db *mssql) TableCheckSql(tableName string) (string, []interface{}) {
 func (db *mssql) GetColumns(tableName string) ([]string, map[string]*core.Column, error) {
 	args := []interface{}{}
 	s := `select a.name as name, b.name as ctype,a.max_length,a.precision,a.scale,a.is_nullable as nullable,
-	      replace(replace(isnull(c.text,''),'(',''),')','') as vdefault   
+	      replace(replace(isnull(c.text,''),'(',''),')','') as vdefault,
-          from sys.columns a left join sys.types b on a.user_type_id=b.user_type_id 
+		  ISNULL(i.is_primary_key, 0)
-          left join  sys.syscomments c  on a.default_object_id=c.id 
+          from sys.columns a 
+		  left join sys.types b on a.user_type_id=b.user_type_id
+          left join sys.syscomments c on a.default_object_id=c.id
+		  LEFT OUTER JOIN 
+    sys.index_columns ic ON ic.object_id = a.object_id AND ic.column_id = a.column_id
+		  LEFT OUTER JOIN 
+    sys.indexes i ON ic.object_id = i.object_id AND ic.index_id = i.index_id
           where a.object_id=object_id('` + tableName + `')`
 	db.LogSQL(s, args)
 
@@ -352,8 +361,8 @@ func (db *mssql) GetColumns(tableName string) ([]string, map[string]*core.Column
 	for rows.Next() {
 		var name, ctype, vdefault string
 		var maxLen, precision, scale int
-		var nullable bool
+		var nullable, isPK bool
-		err = rows.Scan(&name, &ctype, &maxLen, &precision, &scale, &nullable, &vdefault)
+		err = rows.Scan(&name, &ctype, &maxLen, &precision, &scale, &nullable, &vdefault, &isPK)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -363,6 +372,7 @@ func (db *mssql) GetColumns(tableName string) ([]string, map[string]*core.Column
 		col.Name = strings.Trim(name, "` ")
 		col.Nullable = nullable
 		col.Default = vdefault
+		col.IsPrimaryKey = isPK
 		ct := strings.ToUpper(ctype)
 		if ct == "DECIMAL" {
 			col.Length = precision
@@ -468,9 +478,10 @@ WHERE IXS.TYPE_DESC='NONCLUSTERED' and OBJECT_NAME(IXS.OBJECT_ID) =?
 		}
 
 		colName = strings.Trim(colName, "` ")
-
+		var isRegular bool
 		if strings.HasPrefix(indexName, "IDX_"+tableName) || strings.HasPrefix(indexName, "UQE_"+tableName) {
 			indexName = indexName[5+len(tableName):]
+			isRegular = true
 		}
 
 		var index *core.Index
@@ -479,6 +490,7 @@ WHERE IXS.TYPE_DESC='NONCLUSTERED' and OBJECT_NAME(IXS.OBJECT_ID) =?
 			index = new(core.Index)
 			index.Type = indexType
 			index.Name = indexName
+			index.IsRegular = isRegular
 			indexes[indexName] = index
 		}
 		index.AddColumn(colName)
@@ -534,7 +546,6 @@ type odbcDriver struct {
 func (p *odbcDriver) Parse(driverName, dataSourceName string) (*core.Uri, error) {
 	kv := strings.Split(dataSourceName, ";")
 	var dbName string
-
 	for _, c := range kv {
 		vv := strings.Split(strings.TrimSpace(c), "=")
 		if len(vv) == 2 {

vendor/github.com/go-xorm/xorm/dialect_mysql.go

@@ -299,7 +299,7 @@ func (db *mysql) TableCheckSql(tableName string) (string, []interface{}) {
 func (db *mysql) GetColumns(tableName string) ([]string, map[string]*core.Column, error) {
 	args := []interface{}{db.DbName, tableName}
 	s := "SELECT `COLUMN_NAME`, `IS_NULLABLE`, `COLUMN_DEFAULT`, `COLUMN_TYPE`," +
-		" `COLUMN_KEY`, `EXTRA` FROM `INFORMATION_SCHEMA`.`COLUMNS` WHERE `TABLE_SCHEMA` = ? AND `TABLE_NAME` = ?"
+		" `COLUMN_KEY`, `EXTRA`,`COLUMN_COMMENT` FROM `INFORMATION_SCHEMA`.`COLUMNS` WHERE `TABLE_SCHEMA` = ? AND `TABLE_NAME` = ?"
 	db.LogSQL(s, args)
 
 	rows, err := db.DB().Query(s, args...)
@@ -314,13 +314,14 @@ func (db *mysql) GetColumns(tableName string) ([]string, map[string]*core.Column
 		col := new(core.Column)
 		col.Indexes = make(map[string]int)
 
-		var columnName, isNullable, colType, colKey, extra string
+		var columnName, isNullable, colType, colKey, extra, comment string
 		var colDefault *string
-		err = rows.Scan(&columnName, &isNullable, &colDefault, &colType, &colKey, &extra)
+		err = rows.Scan(&columnName, &isNullable, &colDefault, &colType, &colKey, &extra, &comment)
 		if err != nil {
 			return nil, nil, err
 		}
 		col.Name = strings.Trim(columnName, "` ")
+		col.Comment = comment
 		if "YES" == isNullable {
 			col.Nullable = true
 		}
@@ -407,7 +408,7 @@ func (db *mysql) GetColumns(tableName string) ([]string, map[string]*core.Column
 
 func (db *mysql) GetTables() ([]*core.Table, error) {
 	args := []interface{}{db.DbName}
-	s := "SELECT `TABLE_NAME`, `ENGINE`, `TABLE_ROWS`, `AUTO_INCREMENT` from " +
+	s := "SELECT `TABLE_NAME`, `ENGINE`, `TABLE_ROWS`, `AUTO_INCREMENT`, `TABLE_COMMENT` from " +
 		"`INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? AND (`ENGINE`='MyISAM' OR `ENGINE` = 'InnoDB' OR `ENGINE` = 'TokuDB')"
 	db.LogSQL(s, args)
 
@@ -420,14 +421,15 @@ func (db *mysql) GetTables() ([]*core.Table, error) {
 	tables := make([]*core.Table, 0)
 	for rows.Next() {
 		table := core.NewEmptyTable()
-		var name, engine, tableRows string
+		var name, engine, tableRows, comment string
 		var autoIncr *string
-		err = rows.Scan(&name, &engine, &tableRows, &autoIncr)
+		err = rows.Scan(&name, &engine, &tableRows, &autoIncr, &comment)
 		if err != nil {
 			return nil, err
 		}
 
 		table.Name = name
+		table.Comment = comment
 		table.StoreEngine = engine
 		tables = append(tables, table)
 	}

vendor/github.com/go-xorm/xorm/dialect_oracle.go

@@ -824,6 +824,12 @@ func (db *oracle) GetIndexes(tableName string) (map[string]*core.Index, error) {
 
 		indexName = strings.Trim(indexName, `" `)
 
+		var isRegular bool
+		if strings.HasPrefix(indexName, "IDX_"+tableName) || strings.HasPrefix(indexName, "UQE_"+tableName) {
+			indexName = indexName[5+len(tableName):]
+			isRegular = true
+		}
+
 		if uniqueness == "UNIQUE" {
 			indexType = core.UniqueType
 		} else {
@@ -836,6 +842,7 @@ func (db *oracle) GetIndexes(tableName string) (map[string]*core.Index, error) {
 			index = new(core.Index)
 			index.Type = indexType
 			index.Name = indexName
+			index.IsRegular = isRegular
 			indexes[indexName] = index
 		}
 		index.AddColumn(colName)

vendor/github.com/go-xorm/xorm/dialect_postgres.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"sort"
 	"strconv"
 	"strings"
 
@@ -781,6 +780,9 @@ func (db *postgres) SqlType(c *core.Column) string {
 	case core.TinyInt:
 		res = core.SmallInt
 		return res
+	case core.Bit:
+		res = core.Boolean
+		return res
 	case core.MediumInt, core.Int, core.Integer:
 		if c.IsAutoIncrement {
 			return core.Serial
@@ -1078,9 +1080,10 @@ func (db *postgres) GetIndexes(tableName string) (map[string]*core.Index, error)
 		}
 		cs := strings.Split(indexdef, "(")
 		colNames = strings.Split(cs[1][0:len(cs[1])-1], ",")
-
+		var isRegular bool
 		if strings.HasPrefix(indexName, "IDX_"+tableName) || strings.HasPrefix(indexName, "UQE_"+tableName) {
 			newIdxName := indexName[5+len(tableName):]
+			isRegular = true
 			if newIdxName != "" {
 				indexName = newIdxName
 			}
@@ -1090,6 +1093,7 @@ func (db *postgres) GetIndexes(tableName string) (map[string]*core.Index, error)
 		for _, colName := range colNames {
 			index.Cols = append(index.Cols, strings.Trim(colName, `" `))
 		}
+		index.IsRegular = isRegular
 		indexes[index.Name] = index
 	}
 	return indexes, nil
@@ -1112,10 +1116,6 @@ func (vs values) Get(k string) (v string) {
 	return vs[k]
 }
 
-func errorf(s string, args ...interface{}) {
-	panic(fmt.Errorf("pq: %s", fmt.Sprintf(s, args...)))
-}
-
 func parseURL(connstr string) (string, error) {
 	u, err := url.Parse(connstr)
 	if err != nil {
@@ -1126,46 +1126,18 @@ func parseURL(connstr string) (string, error) {
 		return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme)
 	}
 
-	var kvs []string
 	escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`)
-	accrue := func(k, v string) {
-		if v != "" {
-			kvs = append(kvs, k+"="+escaper.Replace(v))
-		}
-	}
-
-	if u.User != nil {
-		v := u.User.Username()
-		accrue("user", v)
-
-		v, _ = u.User.Password()
-		accrue("password", v)
-	}
-
-	i := strings.Index(u.Host, ":")
-	if i < 0 {
-		accrue("host", u.Host)
-	} else {
-		accrue("host", u.Host[:i])
-		accrue("port", u.Host[i+1:])
-	}
 
 	if u.Path != "" {
-		accrue("dbname", u.Path[1:])
+		return escaper.Replace(u.Path[1:]), nil
 	}
 
-	q := u.Query()
+	return "", nil
-	for k := range q {
-		accrue(k, q.Get(k))
-	}
-
-	sort.Strings(kvs) // Makes testing easier (not a performance concern)
-	return strings.Join(kvs, " "), nil
 }
 
-func parseOpts(name string, o values) {
+func parseOpts(name string, o values) error {
 	if len(name) == 0 {
-		return
+		return fmt.Errorf("invalid options: %s", name)
 	}
 
 	name = strings.TrimSpace(name)
@@ -1174,31 +1146,36 @@ func parseOpts(name string, o values) {
 	for _, p := range ps {
 		kv := strings.Split(p, "=")
 		if len(kv) < 2 {
-			errorf("invalid option: %q", p)
+			return fmt.Errorf("invalid option: %q", p)
 		}
 		o.Set(kv[0], kv[1])
 	}
+
+	return nil
 }
 
 func (p *pqDriver) Parse(driverName, dataSourceName string) (*core.Uri, error) {
 	db := &core.Uri{DbType: core.POSTGRES}
-	o := make(values)
 	var err error
+
 	if strings.HasPrefix(dataSourceName, "postgresql://") || strings.HasPrefix(dataSourceName, "postgres://") {
-		dataSourceName, err = parseURL(dataSourceName)
+		db.DbName, err = parseURL(dataSourceName)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		o := make(values)
+		err = parseOpts(dataSourceName, o)
 		if err != nil {
 			return nil, err
 		}
-	}
-	parseOpts(dataSourceName, o)
 
-	db.DbName = o.Get("dbname")
+		db.DbName = o.Get("dbname")
+	}
+
 	if db.DbName == "" {
 		return nil, errors.New("dbname is empty")
 	}
-	/*db.Schema = o.Get("schema")
+
-	if len(db.Schema) == 0 {
-		db.Schema = "public"
-	}*/
 	return db, nil
 }

vendor/github.com/go-xorm/xorm/dialect_sqlite3.go

@@ -14,10 +14,6 @@ import (
 	"github.com/go-xorm/core"
 )
 
-// func init() {
-// 	RegisterDialect("sqlite3", &sqlite3{})
-// }
-
 var (
 	sqlite3ReservedWords = map[string]bool{
 		"ABORT":             true,
@@ -310,11 +306,25 @@ func (db *sqlite3) GetColumns(tableName string) ([]string, map[string]*core.Colu
 	for _, colStr := range colCreates {
 		reg = regexp.MustCompile(`,\s`)
 		colStr = reg.ReplaceAllString(colStr, ",")
+		if strings.HasPrefix(strings.TrimSpace(colStr), "PRIMARY KEY") {
+			parts := strings.Split(strings.TrimSpace(colStr), "(")
+			if len(parts) == 2 {
+				pkCols := strings.Split(strings.TrimRight(strings.TrimSpace(parts[1]), ")"), ",")
+				for _, pk := range pkCols {
+					if col, ok := cols[strings.Trim(strings.TrimSpace(pk), "`")]; ok {
+						col.IsPrimaryKey = true
+					}
+				}
+			}
+			continue
+		}
+
 		fields := strings.Fields(strings.TrimSpace(colStr))
 		col := new(core.Column)
 		col.Indexes = make(map[string]int)
 		col.Nullable = true
 		col.DefaultIsEmpty = true
+
 		for idx, field := range fields {
 			if idx == 0 {
 				col.Name = strings.Trim(strings.Trim(field, "`[] "), `"`)
@@ -405,8 +415,10 @@ func (db *sqlite3) GetIndexes(tableName string) (map[string]*core.Index, error)
 		}
 
 		indexName := strings.Trim(sql[nNStart+6:nNEnd], "` []")
+		var isRegular bool
 		if strings.HasPrefix(indexName, "IDX_"+tableName) || strings.HasPrefix(indexName, "UQE_"+tableName) {
 			index.Name = indexName[5+len(tableName):]
+			isRegular = true
 		} else {
 			index.Name = indexName
 		}
@@ -425,6 +437,7 @@ func (db *sqlite3) GetIndexes(tableName string) (map[string]*core.Index, error)
 		for _, col := range colIndexes {
 			index.Cols = append(index.Cols, strings.Trim(col, "` []"))
 		}
+		index.IsRegular = isRegular
 		indexes[index.Name] = index
 	}
 

vendor/github.com/go-xorm/xorm/doc.go

@@ -8,7 +8,7 @@ Package xorm is a simple and powerful ORM for Go.
 
 Installation
 
-Make sure you have installed Go 1.1+ and then:
+Make sure you have installed Go 1.6+ and then:
 
     go get github.com/go-xorm/xorm
 
@@ -51,11 +51,15 @@ There are 8 major ORM methods and many helpful methods to use to operate databas
     // INSERT INTO struct1 () values ()
     // INSERT INTO struct2 () values (),(),()
 
-2. Query one record from database
+2. Query one record or one variable from database
 
     has, err := engine.Get(&user)
     // SELECT * FROM user LIMIT 1
 
+    var id int64
+    has, err := engine.Table("user").Where("name = ?", name).Get(&id)
+    // SELECT id FROM user WHERE name = ? LIMIT 1
+
 3. Query multiple records from database
 
     var sliceOfStructs []Struct
@@ -86,7 +90,7 @@ another is Rows
 
 5. Update one or more records
 
-    affected, err := engine.Id(...).Update(&user)
+    affected, err := engine.ID(...).Update(&user)
     // UPDATE user SET ...
 
 6. Delete one or more records, Delete MUST has condition
@@ -99,6 +103,9 @@ another is Rows
     counts, err := engine.Count(&user)
     // SELECT count(*) AS total FROM user
 
+    counts, err := engine.SQL("select count(*) FROM user").Count()
+    // select count(*) FROM user
+
 8. Sum records
 
     sumFloat64, err := engine.Sum(&user, "id")

vendor/github.com/go-xorm/xorm/engine.go

@@ -19,6 +19,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/go-xorm/builder"
 	"github.com/go-xorm/core"
 )
 
@@ -40,12 +41,29 @@ type Engine struct {
 	showExecTime bool
 
 	logger     core.ILogger
-	TZLocation *time.Location
+	TZLocation *time.Location // The timezone of the application
 	DatabaseTZ *time.Location // The timezone of the database
 
 	disableGlobalCache bool
 
 	tagHandlers map[string]tagHandler
+
+	engineGroup *EngineGroup
+}
+
+// BufferSize sets buffer size for iterate
+func (engine *Engine) BufferSize(size int) *Session {
+	session := engine.NewSession()
+	session.isAutoClose = true
+	return session.BufferSize(size)
+}
+
+// CondDeleted returns the conditions whether a record is soft deleted.
+func (engine *Engine) CondDeleted(colName string) builder.Cond {
+	if engine.dialect.DBType() == core.MSSQL {
+		return builder.IsNull{colName}
+	}
+	return builder.IsNull{colName}.Or(builder.Eq{colName: zeroTime1})
 }
 
 // ShowSQL show SQL statement or not on logger if log level is great than INFO
@@ -78,6 +96,11 @@ func (engine *Engine) SetLogger(logger core.ILogger) {
 	engine.dialect.SetLogger(logger)
 }
 
+// SetLogLevel sets the logger level
+func (engine *Engine) SetLogLevel(level core.LogLevel) {
+	engine.logger.SetLevel(level)
+}
+
 // SetDisableGlobalCache disable global cache or not
 func (engine *Engine) SetDisableGlobalCache(disable bool) {
 	if engine.disableGlobalCache != disable {
@@ -143,7 +166,6 @@ func (engine *Engine) Quote(value string) string {
 
 // QuoteTo quotes string and writes into the buffer
 func (engine *Engine) QuoteTo(buf *bytes.Buffer, value string) {
-
 	if buf == nil {
 		return
 	}
@@ -169,7 +191,7 @@ func (engine *Engine) quote(sql string) string {
 	return engine.dialect.QuoteStr() + sql + engine.dialect.QuoteStr()
 }
 
-// SqlType will be depracated, please use SQLType instead
+// SqlType will be deprecated, please use SQLType instead
 //
 // Deprecated: use SQLType instead
 func (engine *Engine) SqlType(c *core.Column) string {
@@ -201,26 +223,36 @@ func (engine *Engine) SetDefaultCacher(cacher core.Cacher) {
 	engine.Cacher = cacher
 }
 
+// GetDefaultCacher returns the default cacher
+func (engine *Engine) GetDefaultCacher() core.Cacher {
+	return engine.Cacher
+}
+
 // NoCache If you has set default cacher, and you want temporilly stop use cache,
 // you can use NoCache()
 func (engine *Engine) NoCache() *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.NoCache()
 }
 
 // NoCascade If you do not want to auto cascade load object
 func (engine *Engine) NoCascade() *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.NoCascade()
 }
 
 // MapCacher Set a table use a special cacher
-func (engine *Engine) MapCacher(bean interface{}, cacher core.Cacher) {
+func (engine *Engine) MapCacher(bean interface{}, cacher core.Cacher) error {
 	v := rValue(bean)
-	tb := engine.autoMapType(v)
+	tb, err := engine.autoMapType(v)
+	if err != nil {
+		return err
+	}
+
 	tb.Cacher = cacher
+	return nil
 }
 
 // NewDB provides an interface to operate database directly
@@ -240,7 +272,7 @@ func (engine *Engine) Dialect() core.Dialect {
 
 // NewSession New a session
 func (engine *Engine) NewSession() *Session {
-	session := &Session{Engine: engine}
+	session := &Session{engine: engine}
 	session.Init()
 	return session
 }
@@ -254,7 +286,6 @@ func (engine *Engine) Close() error {
 func (engine *Engine) Ping() error {
 	session := engine.NewSession()
 	defer session.Close()
-	engine.logger.Infof("PING DATABASE %v", engine.DriverName())
 	return session.Ping()
 }
 
@@ -262,43 +293,13 @@ func (engine *Engine) Ping() error {
 func (engine *Engine) logSQL(sqlStr string, sqlArgs ...interface{}) {
 	if engine.showSQL && !engine.showExecTime {
 		if len(sqlArgs) > 0 {
-			engine.logger.Infof("[SQL] %v %v", sqlStr, sqlArgs)
+			engine.logger.Infof("[SQL] %v %#v", sqlStr, sqlArgs)
 		} else {
 			engine.logger.Infof("[SQL] %v", sqlStr)
 		}
 	}
 }
 
-func (engine *Engine) logSQLQueryTime(sqlStr string, args []interface{}, executionBlock func() (*core.Stmt, *core.Rows, error)) (*core.Stmt, *core.Rows, error) {
-	if engine.showSQL && engine.showExecTime {
-		b4ExecTime := time.Now()
-		stmt, res, err := executionBlock()
-		execDuration := time.Since(b4ExecTime)
-		if len(args) > 0 {
-			engine.logger.Infof("[SQL] %s %v - took: %v", sqlStr, args, execDuration)
-		} else {
-			engine.logger.Infof("[SQL] %s - took: %v", sqlStr, execDuration)
-		}
-		return stmt, res, err
-	}
-	return executionBlock()
-}
-
-func (engine *Engine) logSQLExecutionTime(sqlStr string, args []interface{}, executionBlock func() (sql.Result, error)) (sql.Result, error) {
-	if engine.showSQL && engine.showExecTime {
-		b4ExecTime := time.Now()
-		res, err := executionBlock()
-		execDuration := time.Since(b4ExecTime)
-		if len(args) > 0 {
-			engine.logger.Infof("[sql] %s [args] %v - took: %v", sqlStr, args, execDuration)
-		} else {
-			engine.logger.Infof("[sql] %s - took: %v", sqlStr, execDuration)
-		}
-		return res, err
-	}
-	return executionBlock()
-}
-
 // Sql provides raw sql input parameter. When you have a complex SQL statement
 // and cannot use Where, Id, In and etc. Methods to describe, you can use SQL.
 //
@@ -315,7 +316,7 @@ func (engine *Engine) Sql(querystring string, args ...interface{}) *Session {
 // This    code will execute "select * from user" and set the records to users
 func (engine *Engine) SQL(query interface{}, args ...interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.SQL(query, args...)
 }
 
@@ -324,14 +325,14 @@ func (engine *Engine) SQL(query interface{}, args ...interface{}) *Session {
 // invoked. Call NoAutoTime if you dont' want to fill automatically.
 func (engine *Engine) NoAutoTime() *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.NoAutoTime()
 }
 
 // NoAutoCondition disable auto generate Where condition from bean or not
 func (engine *Engine) NoAutoCondition(no ...bool) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.NoAutoCondition(no...)
 }
 
@@ -565,56 +566,56 @@ func (engine *Engine) tbName(v reflect.Value) string {
 // Cascade use cascade or not
 func (engine *Engine) Cascade(trueOrFalse ...bool) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Cascade(trueOrFalse...)
 }
 
 // Where method provide a condition query
 func (engine *Engine) Where(query interface{}, args ...interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Where(query, args...)
 }
 
-// Id will be depracated, please use ID instead
+// Id will be deprecated, please use ID instead
 func (engine *Engine) Id(id interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Id(id)
 }
 
 // ID method provoide a condition as (id) = ?
 func (engine *Engine) ID(id interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.ID(id)
 }
 
 // Before apply before Processor, affected bean is passed to closure arg
 func (engine *Engine) Before(closures func(interface{})) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Before(closures)
 }
 
 // After apply after insert Processor, affected bean is passed to closure arg
 func (engine *Engine) After(closures func(interface{})) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.After(closures)
 }
 
 // Charset set charset when create table, only support mysql now
 func (engine *Engine) Charset(charset string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Charset(charset)
 }
 
 // StoreEngine set store engine when create table, only support mysql now
 func (engine *Engine) StoreEngine(storeEngine string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.StoreEngine(storeEngine)
 }
 
@@ -623,35 +624,35 @@ func (engine *Engine) StoreEngine(storeEngine string) *Session {
 // but distinct will not provide id
 func (engine *Engine) Distinct(columns ...string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Distinct(columns...)
 }
 
 // Select customerize your select columns or contents
 func (engine *Engine) Select(str string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Select(str)
 }
 
 // Cols only use the parameters as select or update columns
 func (engine *Engine) Cols(columns ...string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Cols(columns...)
 }
 
 // AllCols indicates that all columns should be use
 func (engine *Engine) AllCols() *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.AllCols()
 }
 
 // MustCols specify some columns must use even if they are empty
 func (engine *Engine) MustCols(columns ...string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.MustCols(columns...)
 }
 
@@ -662,77 +663,84 @@ func (engine *Engine) MustCols(columns ...string) *Session {
 // it will use parameters's columns
 func (engine *Engine) UseBool(columns ...string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.UseBool(columns...)
 }
 
 // Omit only not use the parameters as select or update columns
 func (engine *Engine) Omit(columns ...string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Omit(columns...)
 }
 
 // Nullable set null when column is zero-value and nullable for update
 func (engine *Engine) Nullable(columns ...string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Nullable(columns...)
 }
 
 // In will generate "column IN (?, ?)"
 func (engine *Engine) In(column string, args ...interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.In(column, args...)
 }
 
+// NotIn will generate "column NOT IN (?, ?)"
+func (engine *Engine) NotIn(column string, args ...interface{}) *Session {
+	session := engine.NewSession()
+	session.isAutoClose = true
+	return session.NotIn(column, args...)
+}
+
 // Incr provides a update string like "column = column + ?"
 func (engine *Engine) Incr(column string, arg ...interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Incr(column, arg...)
 }
 
 // Decr provides a update string like "column = column - ?"
 func (engine *Engine) Decr(column string, arg ...interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Decr(column, arg...)
 }
 
 // SetExpr provides a update string like "column = {expression}"
 func (engine *Engine) SetExpr(column string, expression string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.SetExpr(column, expression)
 }
 
 // Table temporarily change the Get, Find, Update's table
 func (engine *Engine) Table(tableNameOrBean interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Table(tableNameOrBean)
 }
 
 // Alias set the table alias
 func (engine *Engine) Alias(alias string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Alias(alias)
 }
 
 // Limit will generate "LIMIT start, limit"
 func (engine *Engine) Limit(limit int, start ...int) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Limit(limit, start...)
 }
 
 // Desc will generate "ORDER BY column1 DESC, column2 DESC"
 func (engine *Engine) Desc(colNames ...string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Desc(colNames...)
 }
 
@@ -744,39 +752,53 @@ func (engine *Engine) Desc(colNames ...string) *Session {
 //
 func (engine *Engine) Asc(colNames ...string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Asc(colNames...)
 }
 
 // OrderBy will generate "ORDER BY order"
 func (engine *Engine) OrderBy(order string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.OrderBy(order)
 }
 
+// Prepare enables prepare statement
+func (engine *Engine) Prepare() *Session {
+	session := engine.NewSession()
+	session.isAutoClose = true
+	return session.Prepare()
+}
+
 // Join the join_operator should be one of INNER, LEFT OUTER, CROSS etc - this will be prepended to JOIN
 func (engine *Engine) Join(joinOperator string, tablename interface{}, condition string, args ...interface{}) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Join(joinOperator, tablename, condition, args...)
 }
 
 // GroupBy generate group by statement
 func (engine *Engine) GroupBy(keys string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.GroupBy(keys)
 }
 
 // Having generate having statement
 func (engine *Engine) Having(conditions string) *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Having(conditions)
 }
 
-func (engine *Engine) autoMapType(v reflect.Value) *core.Table {
+// UnMapType removes the datbase mapper of a type
+func (engine *Engine) UnMapType(t reflect.Type) {
+	engine.mutex.Lock()
+	defer engine.mutex.Unlock()
+	delete(engine.Tables, t)
+}
+
+func (engine *Engine) autoMapType(v reflect.Value) (*core.Table, error) {
 	t := v.Type()
 	engine.mutex.Lock()
 	defer engine.mutex.Unlock()
@@ -785,24 +807,23 @@ func (engine *Engine) autoMapType(v reflect.Value) *core.Table {
 		var err error
 		table, err = engine.mapType(v)
 		if err != nil {
-			engine.logger.Error(err)
+			return nil, err
-		} else {
+		}
-			engine.Tables[t] = table
+
-			if engine.Cacher != nil {
+		engine.Tables[t] = table
-				if v.CanAddr() {
+		if engine.Cacher != nil {
-					engine.GobRegister(v.Addr().Interface())
+			if v.CanAddr() {
-				} else {
+				engine.GobRegister(v.Addr().Interface())
-					engine.GobRegister(v.Interface())
+			} else {
-				}
+				engine.GobRegister(v.Interface())
 			}
 		}
 	}
-	return table
+	return table, nil
 }
 
 // GobRegister register one struct to gob for cache use
 func (engine *Engine) GobRegister(v interface{}) *Engine {
-	//fmt.Printf("Type: %[1]T => Data: %[1]#v\n", v)
 	gob.Register(v)
 	return engine
 }
@@ -813,10 +834,19 @@ type Table struct {
 	Name string
 }
 
+// IsValid if table is valid
+func (t *Table) IsValid() bool {
+	return t.Table != nil && len(t.Name) > 0
+}
+
 // TableInfo get table info according to bean's content
 func (engine *Engine) TableInfo(bean interface{}) *Table {
 	v := rValue(bean)
-	return &Table{engine.autoMapType(v), engine.tbName(v)}
+	tb, err := engine.autoMapType(v)
+	if err != nil {
+		engine.logger.Error(err)
+	}
+	return &Table{tb, engine.tbName(v)}
 }
 
 func addIndex(indexName string, table *core.Table, col *core.Column, indexType int) {
@@ -911,6 +941,7 @@ func (engine *Engine) mapType(v reflect.Value) (*core.Table, error) {
 
 					k := strings.ToUpper(key)
 					ctx.tagName = k
+					ctx.params = []string{}
 
 					pStart := strings.Index(k, "(")
 					if pStart == 0 {
@@ -918,18 +949,18 @@ func (engine *Engine) mapType(v reflect.Value) (*core.Table, error) {
 					}
 					if pStart > -1 {
 						if !strings.HasSuffix(k, ")") {
-							return nil, errors.New("cannot match ) charactor")
+							return nil, fmt.Errorf("field %s tag %s cannot match ) charactor", col.FieldName, key)
 						}
 
 						ctx.tagName = k[:pStart]
-						ctx.params = strings.Split(k[pStart+1:len(k)-1], ",")
+						ctx.params = strings.Split(key[pStart+1:len(k)-1], ",")
 					}
 
 					if j > 0 {
 						ctx.preTag = strings.ToUpper(tags[j-1])
 					}
 					if j < len(tags)-1 {
-						ctx.nextTag = strings.ToUpper(tags[j+1])
+						ctx.nextTag = tags[j+1]
 					} else {
 						ctx.nextTag = ""
 					}
@@ -993,6 +1024,10 @@ func (engine *Engine) mapType(v reflect.Value) (*core.Table, error) {
 			col = core.NewColumn(engine.ColumnMapper.Obj2Table(t.Field(i).Name),
 				t.Field(i).Name, sqlType, sqlType.DefaultLength,
 				sqlType.DefaultLength2, true)
+
+			if fieldType.Kind() == reflect.Int64 && (strings.ToUpper(col.FieldName) == "ID" || strings.HasSuffix(strings.ToUpper(col.FieldName), ".ID")) {
+				idFieldColName = col.Name
+			}
 		}
 		if col.IsAutoIncrement {
 			col.Nullable = false
@@ -1000,9 +1035,6 @@ func (engine *Engine) mapType(v reflect.Value) (*core.Table, error) {
 
 		table.AddColumn(col)
 
-		if fieldType.Kind() == reflect.Int64 && (strings.ToUpper(col.FieldName) == "ID" || strings.HasSuffix(strings.ToUpper(col.FieldName), ".ID")) {
-			idFieldColName = col.Name
-		}
 	} // end for
 
 	if idFieldColName != "" && len(table.PrimaryKeys) == 0 {
@@ -1066,21 +1098,54 @@ func (engine *Engine) IdOfV(rv reflect.Value) core.PK {
 
 // IDOfV get id from one value of struct
 func (engine *Engine) IDOfV(rv reflect.Value) core.PK {
+	pk, err := engine.idOfV(rv)
+	if err != nil {
+		engine.logger.Error(err)
+		return nil
+	}
+	return pk
+}
+
+func (engine *Engine) idOfV(rv reflect.Value) (core.PK, error) {
 	v := reflect.Indirect(rv)
-	table := engine.autoMapType(v)
+	table, err := engine.autoMapType(v)
+	if err != nil {
+		return nil, err
+	}
+
 	pk := make([]interface{}, len(table.PrimaryKeys))
 	for i, col := range table.PKColumns() {
+		var err error
 		pkField := v.FieldByName(col.FieldName)
 		switch pkField.Kind() {
 		case reflect.String:
-			pk[i] = pkField.String()
+			pk[i], err = engine.idTypeAssertion(col, pkField.String())
 		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-			pk[i] = pkField.Int()
+			pk[i], err = engine.idTypeAssertion(col, strconv.FormatInt(pkField.Int(), 10))
 		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-			pk[i] = pkField.Uint()
+			// id of uint will be converted to int64
+			pk[i], err = engine.idTypeAssertion(col, strconv.FormatUint(pkField.Uint(), 10))
+		}
+
+		if err != nil {
+			return nil, err
 		}
 	}
-	return core.PK(pk)
+	return core.PK(pk), nil
+}
+
+func (engine *Engine) idTypeAssertion(col *core.Column, sid string) (interface{}, error) {
+	if col.SQLType.IsNumeric() {
+		n, err := strconv.ParseInt(sid, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		return n, nil
+	} else if col.SQLType.IsText() {
+		return sid, nil
+	} else {
+		return nil, errors.New("not supported")
+	}
 }
 
 // CreateIndexes create indexes
@@ -1101,13 +1166,6 @@ func (engine *Engine) getCacher2(table *core.Table) core.Cacher {
 	return table.Cacher
 }
 
-func (engine *Engine) getCacher(v reflect.Value) core.Cacher {
-	if table := engine.autoMapType(v); table != nil {
-		return table.Cacher
-	}
-	return engine.Cacher
-}
-
 // ClearCacheBean if enabled cache, clear the cache bean
 func (engine *Engine) ClearCacheBean(bean interface{}, id string) error {
 	v := rValue(bean)
@@ -1116,7 +1174,10 @@ func (engine *Engine) ClearCacheBean(bean interface{}, id string) error {
 		return errors.New("error params")
 	}
 	tableName := engine.tbName(v)
-	table := engine.autoMapType(v)
+	table, err := engine.autoMapType(v)
+	if err != nil {
+		return err
+	}
 	cacher := table.Cacher
 	if cacher == nil {
 		cacher = engine.Cacher
@@ -1137,7 +1198,11 @@ func (engine *Engine) ClearCache(beans ...interface{}) error {
 			return errors.New("error params")
 		}
 		tableName := engine.tbName(v)
-		table := engine.autoMapType(v)
+		table, err := engine.autoMapType(v)
+		if err != nil {
+			return err
+		}
+
 		cacher := table.Cacher
 		if cacher == nil {
 			cacher = engine.Cacher
@@ -1154,19 +1219,23 @@ func (engine *Engine) ClearCache(beans ...interface{}) error {
 // table, column, index, unique. but will not delete or change anything.
 // If you change some field, you should change the database manually.
 func (engine *Engine) Sync(beans ...interface{}) error {
+	session := engine.NewSession()
+	defer session.Close()
+
 	for _, bean := range beans {
 		v := rValue(bean)
 		tableName := engine.tbName(v)
-		table := engine.autoMapType(v)
+		table, err := engine.autoMapType(v)
+		if err != nil {
+			return err
+		}
 
-		s := engine.NewSession()
+		isExist, err := session.Table(bean).isTableExist(tableName)
-		defer s.Close()
-		isExist, err := s.Table(bean).isTableExist(tableName)
 		if err != nil {
 			return err
 		}
 		if !isExist {
-			err = engine.CreateTables(bean)
+			err = session.createTable(bean)
 			if err != nil {
 				return err
 			}
@@ -1177,11 +1246,11 @@ func (engine *Engine) Sync(beans ...interface{}) error {
 		  }*/
 		var isEmpty bool
 		if isEmpty {
-			err = engine.DropTables(bean)
+			err = session.dropTable(bean)
 			if err != nil {
 				return err
 			}
-			err = engine.CreateTables(bean)
+			err = session.createTable(bean)
 			if err != nil {
 				return err
 			}
@@ -1192,9 +1261,9 @@ func (engine *Engine) Sync(beans ...interface{}) error {
 					return err
 				}
 				if !isExist {
-					session := engine.NewSession()
+					if err := session.statement.setRefValue(v); err != nil {
-					session.Statement.setRefValue(v)
+						return err
-					defer session.Close()
+					}
 					err = session.addColumn(col.Name)
 					if err != nil {
 						return err
@@ -1203,19 +1272,19 @@ func (engine *Engine) Sync(beans ...interface{}) error {
 			}
 
 			for name, index := range table.Indexes {
-				session := engine.NewSession()
+				if err := session.statement.setRefValue(v); err != nil {
-				session.Statement.setRefValue(v)
+					return err
-				defer session.Close()
+				}
 				if index.Type == core.UniqueType {
-					//isExist, err := session.isIndexExist(table.Name, name, true)
 					isExist, err := session.isIndexExist2(tableName, index.Cols, true)
 					if err != nil {
 						return err
 					}
 					if !isExist {
-						session := engine.NewSession()
+						if err := session.statement.setRefValue(v); err != nil {
-						session.Statement.setRefValue(v)
+							return err
-						defer session.Close()
+						}
+
 						err = session.addUnique(tableName, name)
 						if err != nil {
 							return err
@@ -1227,9 +1296,10 @@ func (engine *Engine) Sync(beans ...interface{}) error {
 						return err
 					}
 					if !isExist {
-						session := engine.NewSession()
+						if err := session.statement.setRefValue(v); err != nil {
-						session.Statement.setRefValue(v)
+							return err
-						defer session.Close()
+						}
+
 						err = session.addIndex(tableName, name)
 						if err != nil {
 							return err
@@ -1251,35 +1321,6 @@ func (engine *Engine) Sync2(beans ...interface{}) error {
 	return s.Sync2(beans...)
 }
 
-func (engine *Engine) unMap(beans ...interface{}) (e error) {
-	engine.mutex.Lock()
-	defer engine.mutex.Unlock()
-	for _, bean := range beans {
-		t := rType(bean)
-		if _, ok := engine.Tables[t]; ok {
-			delete(engine.Tables, t)
-		}
-	}
-	return
-}
-
-// Drop all mapped table
-func (engine *Engine) dropAll() error {
-	session := engine.NewSession()
-	defer session.Close()
-
-	err := session.Begin()
-	if err != nil {
-		return err
-	}
-	err = session.dropAll()
-	if err != nil {
-		session.Rollback()
-		return err
-	}
-	return session.Commit()
-}
-
 // CreateTables create tabls according bean
 func (engine *Engine) CreateTables(beans ...interface{}) error {
 	session := engine.NewSession()
@@ -1291,7 +1332,7 @@ func (engine *Engine) CreateTables(beans ...interface{}) error {
 	}
 
 	for _, bean := range beans {
-		err = session.CreateTable(bean)
+		err = session.createTable(bean)
 		if err != nil {
 			session.Rollback()
 			return err
@@ -1311,7 +1352,7 @@ func (engine *Engine) DropTables(beans ...interface{}) error {
 	}
 
 	for _, bean := range beans {
-		err = session.DropTable(bean)
+		err = session.dropTable(bean)
 		if err != nil {
 			session.Rollback()
 			return err
@@ -1320,10 +1361,11 @@ func (engine *Engine) DropTables(beans ...interface{}) error {
 	return session.Commit()
 }
 
-func (engine *Engine) createAll() error {
+// DropIndexes drop indexes of a table
+func (engine *Engine) DropIndexes(bean interface{}) error {
 	session := engine.NewSession()
 	defer session.Close()
-	return session.createAll()
+	return session.DropIndexes(bean)
 }
 
 // Exec raw sql
@@ -1334,10 +1376,24 @@ func (engine *Engine) Exec(sql string, args ...interface{}) (sql.Result, error)
 }
 
 // Query a raw sql and return records as []map[string][]byte
-func (engine *Engine) Query(sql string, paramStr ...interface{}) (resultsSlice []map[string][]byte, err error) {
+func (engine *Engine) Query(sqlorArgs ...interface{}) (resultsSlice []map[string][]byte, err error) {
 	session := engine.NewSession()
 	defer session.Close()
-	return session.Query(sql, paramStr...)
+	return session.Query(sqlorArgs...)
+}
+
+// QueryString runs a raw sql and return records as []map[string]string
+func (engine *Engine) QueryString(sqlorArgs ...interface{}) ([]map[string]string, error) {
+	session := engine.NewSession()
+	defer session.Close()
+	return session.QueryString(sqlorArgs...)
+}
+
+// QueryInterface runs a raw sql and return records as []map[string]interface{}
+func (engine *Engine) QueryInterface(sqlorArgs ...interface{}) ([]map[string]interface{}, error) {
+	session := engine.NewSession()
+	defer session.Close()
+	return session.QueryInterface(sqlorArgs...)
 }
 
 // Insert one or more records
@@ -1381,6 +1437,13 @@ func (engine *Engine) Get(bean interface{}) (bool, error) {
 	return session.Get(bean)
 }
 
+// Exist returns true if the record exist otherwise return false
+func (engine *Engine) Exist(bean ...interface{}) (bool, error) {
+	session := engine.NewSession()
+	defer session.Close()
+	return session.Exist(bean...)
+}
+
 // Find retrieve records from table, condiBeans's non-empty fields
 // are conditions. beans could be []Struct, []*Struct, map[int64]Struct
 // map[int64]*Struct
@@ -1406,10 +1469,10 @@ func (engine *Engine) Rows(bean interface{}) (*Rows, error) {
 }
 
 // Count counts the records. bean's non-empty fields are conditions.
-func (engine *Engine) Count(bean interface{}) (int64, error) {
+func (engine *Engine) Count(bean ...interface{}) (int64, error) {
 	session := engine.NewSession()
 	defer session.Close()
-	return session.Count(bean)
+	return session.Count(bean...)
 }
 
 // Sum sum the records by some column. bean's non-empty fields are conditions.
@@ -1419,6 +1482,13 @@ func (engine *Engine) Sum(bean interface{}, colName string) (float64, error) {
 	return session.Sum(bean, colName)
 }
 
+// SumInt sum the records by some column. bean's non-empty fields are conditions.
+func (engine *Engine) SumInt(bean interface{}, colName string) (int64, error) {
+	session := engine.NewSession()
+	defer session.Close()
+	return session.SumInt(bean, colName)
+}
+
 // Sums sum the records by some columns. bean's non-empty fields are conditions.
 func (engine *Engine) Sums(bean interface{}, colNames ...string) ([]float64, error) {
 	session := engine.NewSession()
@@ -1474,7 +1544,6 @@ func (engine *Engine) Import(r io.Reader) ([]sql.Result, error) {
 			results = append(results, result)
 			if err != nil {
 				return nil, err
-				//lastError = err
 			}
 		}
 	}
@@ -1482,49 +1551,32 @@ func (engine *Engine) Import(r io.Reader) ([]sql.Result, error) {
 	return results, lastError
 }
 
-// TZTime change one time to xorm time location
+// nowTime return current time
-func (engine *Engine) TZTime(t time.Time) time.Time {
+func (engine *Engine) nowTime(col *core.Column) (interface{}, time.Time) {
-	if !t.IsZero() { // if time is not initialized it's not suitable for Time.In()
+	t := time.Now()
-		return t.In(engine.TZLocation)
+	var tz = engine.DatabaseTZ
+	if !col.DisableTimeZone && col.TimeZone != nil {
+		tz = col.TimeZone
 	}
-	return t
+	return engine.formatTime(col.SQLType.Name, t.In(tz)), t.In(engine.TZLocation)
-}
-
-// NowTime return current time
-func (engine *Engine) NowTime(sqlTypeName string) interface{} {
-	t := time.Now()
-	return engine.FormatTime(sqlTypeName, t)
-}
-
-// NowTime2 return current time
-func (engine *Engine) NowTime2(sqlTypeName string) (interface{}, time.Time) {
-	t := time.Now()
-	return engine.FormatTime(sqlTypeName, t), t
-}
-
-// FormatTime format time
-func (engine *Engine) FormatTime(sqlTypeName string, t time.Time) (v interface{}) {
-	return engine.formatTime(engine.TZLocation, sqlTypeName, t)
 }
 
 func (engine *Engine) formatColTime(col *core.Column, t time.Time) (v interface{}) {
-	if col.DisableTimeZone {
+	if t.IsZero() {
-		return engine.formatTime(nil, col.SQLType.Name, t)
+		if col.Nullable {
-	} else if col.TimeZone != nil {
+			return nil
-		return engine.formatTime(col.TimeZone, col.SQLType.Name, t)
+		}
+		return ""
 	}
-	return engine.formatTime(engine.TZLocation, col.SQLType.Name, t)
+
+	if col.TimeZone != nil {
+		return engine.formatTime(col.SQLType.Name, t.In(col.TimeZone))
+	}
+	return engine.formatTime(col.SQLType.Name, t.In(engine.DatabaseTZ))
 }
 
-func (engine *Engine) formatTime(tz *time.Location, sqlTypeName string, t time.Time) (v interface{}) {
+// formatTime format time as column type
-	if engine.dialect.DBType() == core.ORACLE {
+func (engine *Engine) formatTime(sqlTypeName string, t time.Time) (v interface{}) {
-		return t
-	}
-	if tz != nil {
-		t = t.In(tz)
-	} else {
-		t = engine.TZTime(t)
-	}
 	switch sqlTypeName {
 	case core.Time:
 		s := t.Format("2006-01-02 15:04:05") //time.RFC3339
@@ -1532,18 +1584,10 @@ func (engine *Engine) formatTime(tz *time.Location, sqlTypeName string, t time.T
 	case core.Date:
 		v = t.Format("2006-01-02")
 	case core.DateTime, core.TimeStamp:
-		if engine.dialect.DBType() == "ql" {
+		v = t.Format("2006-01-02 15:04:05")
-			v = t
-		} else if engine.dialect.DBType() == "sqlite3" {
-			v = t.UTC().Format("2006-01-02 15:04:05")
-		} else {
-			v = t.Format("2006-01-02 15:04:05")
-		}
 	case core.TimeStampz:
 		if engine.dialect.DBType() == core.MSSQL {
 			v = t.Format("2006-01-02T15:04:05.9999999Z07:00")
-		} else if engine.DriverName() == "mssql" {
-			v = t
 		} else {
 			v = t.Format(time.RFC3339Nano)
 		}
@@ -1555,9 +1599,39 @@ func (engine *Engine) formatTime(tz *time.Location, sqlTypeName string, t time.T
 	return
 }
 
+// GetColumnMapper returns the column name mapper
+func (engine *Engine) GetColumnMapper() core.IMapper {
+	return engine.ColumnMapper
+}
+
+// GetTableMapper returns the table name mapper
+func (engine *Engine) GetTableMapper() core.IMapper {
+	return engine.TableMapper
+}
+
+// GetTZLocation returns time zone of the application
+func (engine *Engine) GetTZLocation() *time.Location {
+	return engine.TZLocation
+}
+
+// SetTZLocation sets time zone of the application
+func (engine *Engine) SetTZLocation(tz *time.Location) {
+	engine.TZLocation = tz
+}
+
+// GetTZDatabase returns time zone of the database
+func (engine *Engine) GetTZDatabase() *time.Location {
+	return engine.DatabaseTZ
+}
+
+// SetTZDatabase sets time zone of the database
+func (engine *Engine) SetTZDatabase(tz *time.Location) {
+	engine.DatabaseTZ = tz
+}
+
 // Unscoped always disable struct tag "deleted"
 func (engine *Engine) Unscoped() *Session {
 	session := engine.NewSession()
-	session.IsAutoClose = true
+	session.isAutoClose = true
 	return session.Unscoped()
 }

vendor/github.com/go-xorm/xorm/engine_cond.go

@@ -0,0 +1,230 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package xorm
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"time"
+
+	"github.com/go-xorm/builder"
+	"github.com/go-xorm/core"
+)
+
+func (engine *Engine) buildConds(table *core.Table, bean interface{},
+	includeVersion bool, includeUpdated bool, includeNil bool,
+	includeAutoIncr bool, allUseBool bool, useAllCols bool, unscoped bool,
+	mustColumnMap map[string]bool, tableName, aliasName string, addedTableName bool) (builder.Cond, error) {
+	var conds []builder.Cond
+	for _, col := range table.Columns() {
+		if !includeVersion && col.IsVersion {
+			continue
+		}
+		if !includeUpdated && col.IsUpdated {
+			continue
+		}
+		if !includeAutoIncr && col.IsAutoIncrement {
+			continue
+		}
+
+		if engine.dialect.DBType() == core.MSSQL && (col.SQLType.Name == core.Text || col.SQLType.IsBlob() || col.SQLType.Name == core.TimeStampz) {
+			continue
+		}
+		if col.SQLType.IsJson() {
+			continue
+		}
+
+		var colName string
+		if addedTableName {
+			var nm = tableName
+			if len(aliasName) > 0 {
+				nm = aliasName
+			}
+			colName = engine.Quote(nm) + "." + engine.Quote(col.Name)
+		} else {
+			colName = engine.Quote(col.Name)
+		}
+
+		fieldValuePtr, err := col.ValueOf(bean)
+		if err != nil {
+			engine.logger.Error(err)
+			continue
+		}
+
+		if col.IsDeleted && !unscoped { // tag "deleted" is enabled
+			conds = append(conds, engine.CondDeleted(colName))
+		}
+
+		fieldValue := *fieldValuePtr
+		if fieldValue.Interface() == nil {
+			continue
+		}
+
+		fieldType := reflect.TypeOf(fieldValue.Interface())
+		requiredField := useAllCols
+
+		if b, ok := getFlagForColumn(mustColumnMap, col); ok {
+			if b {
+				requiredField = true
+			} else {
+				continue
+			}
+		}
+
+		if fieldType.Kind() == reflect.Ptr {
+			if fieldValue.IsNil() {
+				if includeNil {
+					conds = append(conds, builder.Eq{colName: nil})
+				}
+				continue
+			} else if !fieldValue.IsValid() {
+				continue
+			} else {
+				// dereference ptr type to instance type
+				fieldValue = fieldValue.Elem()
+				fieldType = reflect.TypeOf(fieldValue.Interface())
+				requiredField = true
+			}
+		}
+
+		var val interface{}
+		switch fieldType.Kind() {
+		case reflect.Bool:
+			if allUseBool || requiredField {
+				val = fieldValue.Interface()
+			} else {
+				// if a bool in a struct, it will not be as a condition because it default is false,
+				// please use Where() instead
+				continue
+			}
+		case reflect.String:
+			if !requiredField && fieldValue.String() == "" {
+				continue
+			}
+			// for MyString, should convert to string or panic
+			if fieldType.String() != reflect.String.String() {
+				val = fieldValue.String()
+			} else {
+				val = fieldValue.Interface()
+			}
+		case reflect.Int8, reflect.Int16, reflect.Int, reflect.Int32, reflect.Int64:
+			if !requiredField && fieldValue.Int() == 0 {
+				continue
+			}
+			val = fieldValue.Interface()
+		case reflect.Float32, reflect.Float64:
+			if !requiredField && fieldValue.Float() == 0.0 {
+				continue
+			}
+			val = fieldValue.Interface()
+		case reflect.Uint8, reflect.Uint16, reflect.Uint, reflect.Uint32, reflect.Uint64:
+			if !requiredField && fieldValue.Uint() == 0 {
+				continue
+			}
+			t := int64(fieldValue.Uint())
+			val = reflect.ValueOf(&t).Interface()
+		case reflect.Struct:
+			if fieldType.ConvertibleTo(core.TimeType) {
+				t := fieldValue.Convert(core.TimeType).Interface().(time.Time)
+				if !requiredField && (t.IsZero() || !fieldValue.IsValid()) {
+					continue
+				}
+				val = engine.formatColTime(col, t)
+			} else if _, ok := reflect.New(fieldType).Interface().(core.Conversion); ok {
+				continue
+			} else if valNul, ok := fieldValue.Interface().(driver.Valuer); ok {
+				val, _ = valNul.Value()
+				if val == nil {
+					continue
+				}
+			} else {
+				if col.SQLType.IsJson() {
+					if col.SQLType.IsText() {
+						bytes, err := json.Marshal(fieldValue.Interface())
+						if err != nil {
+							engine.logger.Error(err)
+							continue
+						}
+						val = string(bytes)
+					} else if col.SQLType.IsBlob() {
+						var bytes []byte
+						var err error
+						bytes, err = json.Marshal(fieldValue.Interface())
+						if err != nil {
+							engine.logger.Error(err)
+							continue
+						}
+						val = bytes
+					}
+				} else {
+					engine.autoMapType(fieldValue)
+					if table, ok := engine.Tables[fieldValue.Type()]; ok {
+						if len(table.PrimaryKeys) == 1 {
+							pkField := reflect.Indirect(fieldValue).FieldByName(table.PKColumns()[0].FieldName)
+							// fix non-int pk issues
+							//if pkField.Int() != 0 {
+							if pkField.IsValid() && !isZero(pkField.Interface()) {
+								val = pkField.Interface()
+							} else {
+								continue
+							}
+						} else {
+							//TODO: how to handler?
+							return nil, fmt.Errorf("not supported %v as %v", fieldValue.Interface(), table.PrimaryKeys)
+						}
+					} else {
+						val = fieldValue.Interface()
+					}
+				}
+			}
+		case reflect.Array:
+			continue
+		case reflect.Slice, reflect.Map:
+			if fieldValue == reflect.Zero(fieldType) {
+				continue
+			}
+			if fieldValue.IsNil() || !fieldValue.IsValid() || fieldValue.Len() == 0 {
+				continue
+			}
+
+			if col.SQLType.IsText() {
+				bytes, err := json.Marshal(fieldValue.Interface())
+				if err != nil {
+					engine.logger.Error(err)
+					continue
+				}
+				val = string(bytes)
+			} else if col.SQLType.IsBlob() {
+				var bytes []byte
+				var err error
+				if (fieldType.Kind() == reflect.Array || fieldType.Kind() == reflect.Slice) &&
+					fieldType.Elem().Kind() == reflect.Uint8 {
+					if fieldValue.Len() > 0 {
+						val = fieldValue.Bytes()
+					} else {
+						continue
+					}
+				} else {
+					bytes, err = json.Marshal(fieldValue.Interface())
+					if err != nil {
+						engine.logger.Error(err)
+						continue
+					}
+					val = bytes
+				}
+			} else {
+				continue
+			}
+		default:
+			val = fieldValue.Interface()
+		}
+
+		conds = append(conds, builder.Eq{colName: val})
+	}
+
+	return builder.And(conds...), nil
+}

vendor/github.com/go-xorm/xorm/engine_group.go

@@ -0,0 +1,194 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package xorm
+
+import (
+	"github.com/go-xorm/core"
+)
+
+// EngineGroup defines an engine group
+type EngineGroup struct {
+	*Engine
+	slaves []*Engine
+	policy GroupPolicy
+}
+
+// NewEngineGroup creates a new engine group
+func NewEngineGroup(args1 interface{}, args2 interface{}, policies ...GroupPolicy) (*EngineGroup, error) {
+	var eg EngineGroup
+	if len(policies) > 0 {
+		eg.policy = policies[0]
+	} else {
+		eg.policy = RoundRobinPolicy()
+	}
+
+	driverName, ok1 := args1.(string)
+	conns, ok2 := args2.([]string)
+	if ok1 && ok2 {
+		engines := make([]*Engine, len(conns))
+		for i, conn := range conns {
+			engine, err := NewEngine(driverName, conn)
+			if err != nil {
+				return nil, err
+			}
+			engine.engineGroup = &eg
+			engines[i] = engine
+		}
+
+		eg.Engine = engines[0]
+		eg.slaves = engines[1:]
+		return &eg, nil
+	}
+
+	master, ok3 := args1.(*Engine)
+	slaves, ok4 := args2.([]*Engine)
+	if ok3 && ok4 {
+		master.engineGroup = &eg
+		for i := 0; i < len(slaves); i++ {
+			slaves[i].engineGroup = &eg
+		}
+		eg.Engine = master
+		eg.slaves = slaves
+		return &eg, nil
+	}
+	return nil, ErrParamsType
+}
+
+// Close the engine
+func (eg *EngineGroup) Close() error {
+	err := eg.Engine.Close()
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < len(eg.slaves); i++ {
+		err := eg.slaves[i].Close()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Master returns the master engine
+func (eg *EngineGroup) Master() *Engine {
+	return eg.Engine
+}
+
+// Ping tests if database is alive
+func (eg *EngineGroup) Ping() error {
+	if err := eg.Engine.Ping(); err != nil {
+		return err
+	}
+
+	for _, slave := range eg.slaves {
+		if err := slave.Ping(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// SetColumnMapper set the column name mapping rule
+func (eg *EngineGroup) SetColumnMapper(mapper core.IMapper) {
+	eg.Engine.ColumnMapper = mapper
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].ColumnMapper = mapper
+	}
+}
+
+// SetDefaultCacher set the default cacher
+func (eg *EngineGroup) SetDefaultCacher(cacher core.Cacher) {
+	eg.Engine.SetDefaultCacher(cacher)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].SetDefaultCacher(cacher)
+	}
+}
+
+// SetLogger set the new logger
+func (eg *EngineGroup) SetLogger(logger core.ILogger) {
+	eg.Engine.SetLogger(logger)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].SetLogger(logger)
+	}
+}
+
+// SetLogLevel sets the logger level
+func (eg *EngineGroup) SetLogLevel(level core.LogLevel) {
+	eg.Engine.SetLogLevel(level)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].SetLogLevel(level)
+	}
+}
+
+// SetMapper set the name mapping rules
+func (eg *EngineGroup) SetMapper(mapper core.IMapper) {
+	eg.Engine.SetMapper(mapper)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].SetMapper(mapper)
+	}
+}
+
+// SetMaxIdleConns set the max idle connections on pool, default is 2
+func (eg *EngineGroup) SetMaxIdleConns(conns int) {
+	eg.Engine.db.SetMaxIdleConns(conns)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].db.SetMaxIdleConns(conns)
+	}
+}
+
+// SetMaxOpenConns is only available for go 1.2+
+func (eg *EngineGroup) SetMaxOpenConns(conns int) {
+	eg.Engine.db.SetMaxOpenConns(conns)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].db.SetMaxOpenConns(conns)
+	}
+}
+
+// SetPolicy set the group policy
+func (eg *EngineGroup) SetPolicy(policy GroupPolicy) *EngineGroup {
+	eg.policy = policy
+	return eg
+}
+
+// SetTableMapper set the table name mapping rule
+func (eg *EngineGroup) SetTableMapper(mapper core.IMapper) {
+	eg.Engine.TableMapper = mapper
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].TableMapper = mapper
+	}
+}
+
+// ShowExecTime show SQL statement and execute time or not on logger if log level is great than INFO
+func (eg *EngineGroup) ShowExecTime(show ...bool) {
+	eg.Engine.ShowExecTime(show...)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].ShowExecTime(show...)
+	}
+}
+
+// ShowSQL show SQL statement or not on logger if log level is great than INFO
+func (eg *EngineGroup) ShowSQL(show ...bool) {
+	eg.Engine.ShowSQL(show...)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].ShowSQL(show...)
+	}
+}
+
+// Slave returns one of the physical databases which is a slave according the policy
+func (eg *EngineGroup) Slave() *Engine {
+	switch len(eg.slaves) {
+	case 0:
+		return eg.Engine
+	case 1:
+		return eg.slaves[0]
+	}
+	return eg.policy.Slave(eg)
+}
+
+// Slaves returns all the slaves
+func (eg *EngineGroup) Slaves() []*Engine {
+	return eg.slaves
+}

vendor/github.com/go-xorm/xorm/engine_group_policy.go

@@ -0,0 +1,116 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package xorm
+
+import (
+	"math/rand"
+	"sync"
+	"time"
+)
+
+// GroupPolicy is be used by chosing the current slave from slaves
+type GroupPolicy interface {
+	Slave(*EngineGroup) *Engine
+}
+
+// GroupPolicyHandler should be used when a function is a GroupPolicy
+type GroupPolicyHandler func(*EngineGroup) *Engine
+
+// Slave implements the chosen of slaves
+func (h GroupPolicyHandler) Slave(eg *EngineGroup) *Engine {
+	return h(eg)
+}
+
+// RandomPolicy implmentes randomly chose the slave of slaves
+func RandomPolicy() GroupPolicyHandler {
+	var r = rand.New(rand.NewSource(time.Now().UnixNano()))
+	return func(g *EngineGroup) *Engine {
+		return g.Slaves()[r.Intn(len(g.Slaves()))]
+	}
+}
+
+// WeightRandomPolicy implmentes randomly chose the slave of slaves
+func WeightRandomPolicy(weights []int) GroupPolicyHandler {
+	var rands = make([]int, 0, len(weights))
+	for i := 0; i < len(weights); i++ {
+		for n := 0; n < weights[i]; n++ {
+			rands = append(rands, i)
+		}
+	}
+	var r = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	return func(g *EngineGroup) *Engine {
+		var slaves = g.Slaves()
+		idx := rands[r.Intn(len(rands))]
+		if idx >= len(slaves) {
+			idx = len(slaves) - 1
+		}
+		return slaves[idx]
+	}
+}
+
+func RoundRobinPolicy() GroupPolicyHandler {
+	var pos = -1
+	var lock sync.Mutex
+	return func(g *EngineGroup) *Engine {
+		var slaves = g.Slaves()
+
+		lock.Lock()
+		defer lock.Unlock()
+		pos++
+		if pos >= len(slaves) {
+			pos = 0
+		}
+
+		return slaves[pos]
+	}
+}
+
+func WeightRoundRobinPolicy(weights []int) GroupPolicyHandler {
+	var rands = make([]int, 0, len(weights))
+	for i := 0; i < len(weights); i++ {
+		for n := 0; n < weights[i]; n++ {
+			rands = append(rands, i)
+		}
+	}
+	var pos = -1
+	var lock sync.Mutex
+
+	return func(g *EngineGroup) *Engine {
+		var slaves = g.Slaves()
+		lock.Lock()
+		defer lock.Unlock()
+		pos++
+		if pos >= len(rands) {
+			pos = 0
+		}
+
+		idx := rands[pos]
+		if idx >= len(slaves) {
+			idx = len(slaves) - 1
+		}
+		return slaves[idx]
+	}
+}
+
+// LeastConnPolicy implements GroupPolicy, every time will get the least connections slave
+func LeastConnPolicy() GroupPolicyHandler {
+	return func(g *EngineGroup) *Engine {
+		var slaves = g.Slaves()
+		connections := 0
+		idx := 0
+		for i := 0; i < len(slaves); i++ {
+			openConnections := slaves[i].DB().Stats().OpenConnections
+			if i == 0 {
+				connections = openConnections
+				idx = i
+			} else if openConnections <= connections {
+				connections = openConnections
+				idx = i
+			}
+		}
+		return slaves[idx]
+	}
+}

vendor/github.com/go-xorm/xorm/engine_maxlife.go

@@ -0,0 +1,22 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.6
+
+package xorm
+
+import "time"
+
+// SetConnMaxLifetime sets the maximum amount of time a connection may be reused.
+func (engine *Engine) SetConnMaxLifetime(d time.Duration) {
+	engine.db.SetConnMaxLifetime(d)
+}
+
+// SetConnMaxLifetime sets the maximum amount of time a connection may be reused.
+func (eg *EngineGroup) SetConnMaxLifetime(d time.Duration) {
+	eg.Engine.SetConnMaxLifetime(d)
+	for i := 0; i < len(eg.slaves); i++ {
+		eg.slaves[i].SetConnMaxLifetime(d)
+	}
+}

vendor/github.com/go-xorm/xorm/error.go

@@ -23,4 +23,6 @@ var (
 	ErrNeedDeletedCond = errors.New("Delete need at least one condition")
 	// ErrNotImplemented not implemented
 	ErrNotImplemented = errors.New("Not implemented")
+	// ErrConditionType condition type unsupported
+	ErrConditionType = errors.New("Unsupported conditon type")
 )

vendor/github.com/go-xorm/xorm/helpers.go

@@ -196,25 +196,43 @@ func isArrayValueZero(v reflect.Value) bool {
 
 func int64ToIntValue(id int64, tp reflect.Type) reflect.Value {
 	var v interface{}
-	switch tp.Kind() {
+	kind := tp.Kind()
-	case reflect.Int16:
+
-		v = int16(id)
+	if kind == reflect.Ptr {
-	case reflect.Int32:
+		kind = tp.Elem().Kind()
-		v = int32(id)
-	case reflect.Int:
-		v = int(id)
-	case reflect.Int64:
-		v = id
-	case reflect.Uint16:
-		v = uint16(id)
-	case reflect.Uint32:
-		v = uint32(id)
-	case reflect.Uint64:
-		v = uint64(id)
-	case reflect.Uint:
-		v = uint(id)
 	}
-	return reflect.ValueOf(v).Convert(tp)
+
+	switch kind {
+	case reflect.Int16:
+		temp := int16(id)
+		v = &temp
+	case reflect.Int32:
+		temp := int32(id)
+		v = &temp
+	case reflect.Int:
+		temp := int(id)
+		v = &temp
+	case reflect.Int64:
+		temp := id
+		v = &temp
+	case reflect.Uint16:
+		temp := uint16(id)
+		v = &temp
+	case reflect.Uint32:
+		temp := uint32(id)
+		v = &temp
+	case reflect.Uint64:
+		temp := uint64(id)
+		v = &temp
+	case reflect.Uint:
+		temp := uint(id)
+		v = &temp
+	}
+
+	if tp.Kind() == reflect.Ptr {
+		return reflect.ValueOf(v).Convert(tp)
+	}
+	return reflect.ValueOf(v).Elem().Convert(tp)
 }
 
 func int64ToInt(id int64, tp reflect.Type) interface{} {
@@ -302,180 +320,6 @@ func sliceEq(left, right []string) bool {
 	return true
 }
 
-func reflect2value(rawValue *reflect.Value) (str string, err error) {
-	aa := reflect.TypeOf((*rawValue).Interface())
-	vv := reflect.ValueOf((*rawValue).Interface())
-	switch aa.Kind() {
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		str = strconv.FormatInt(vv.Int(), 10)
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-		str = strconv.FormatUint(vv.Uint(), 10)
-	case reflect.Float32, reflect.Float64:
-		str = strconv.FormatFloat(vv.Float(), 'f', -1, 64)
-	case reflect.String:
-		str = vv.String()
-	case reflect.Array, reflect.Slice:
-		switch aa.Elem().Kind() {
-		case reflect.Uint8:
-			data := rawValue.Interface().([]byte)
-			str = string(data)
-		default:
-			err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
-		}
-	// time type
-	case reflect.Struct:
-		if aa.ConvertibleTo(core.TimeType) {
-			str = vv.Convert(core.TimeType).Interface().(time.Time).Format(time.RFC3339Nano)
-		} else {
-			err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
-		}
-	case reflect.Bool:
-		str = strconv.FormatBool(vv.Bool())
-	case reflect.Complex128, reflect.Complex64:
-		str = fmt.Sprintf("%v", vv.Complex())
-	/* TODO: unsupported types below
-	   case reflect.Map:
-	   case reflect.Ptr:
-	   case reflect.Uintptr:
-	   case reflect.UnsafePointer:
-	   case reflect.Chan, reflect.Func, reflect.Interface:
-	*/
-	default:
-		err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
-	}
-	return
-}
-
-func value2Bytes(rawValue *reflect.Value) (data []byte, err error) {
-	var str string
-	str, err = reflect2value(rawValue)
-	if err != nil {
-		return
-	}
-	data = []byte(str)
-	return
-}
-
-func value2String(rawValue *reflect.Value) (data string, err error) {
-	data, err = reflect2value(rawValue)
-	if err != nil {
-		return
-	}
-	return
-}
-
-func rows2Strings(rows *core.Rows) (resultsSlice []map[string]string, err error) {
-	fields, err := rows.Columns()
-	if err != nil {
-		return nil, err
-	}
-	for rows.Next() {
-		result, err := row2mapStr(rows, fields)
-		if err != nil {
-			return nil, err
-		}
-		resultsSlice = append(resultsSlice, result)
-	}
-
-	return resultsSlice, nil
-}
-
-func rows2maps(rows *core.Rows) (resultsSlice []map[string][]byte, err error) {
-	fields, err := rows.Columns()
-	if err != nil {
-		return nil, err
-	}
-	for rows.Next() {
-		result, err := row2map(rows, fields)
-		if err != nil {
-			return nil, err
-		}
-		resultsSlice = append(resultsSlice, result)
-	}
-
-	return resultsSlice, nil
-}
-
-func row2map(rows *core.Rows, fields []string) (resultsMap map[string][]byte, err error) {
-	result := make(map[string][]byte)
-	scanResultContainers := make([]interface{}, len(fields))
-	for i := 0; i < len(fields); i++ {
-		var scanResultContainer interface{}
-		scanResultContainers[i] = &scanResultContainer
-	}
-	if err := rows.Scan(scanResultContainers...); err != nil {
-		return nil, err
-	}
-
-	for ii, key := range fields {
-		rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
-		//if row is null then ignore
-		if rawValue.Interface() == nil {
-			//fmt.Println("ignore ...", key, rawValue)
-			continue
-		}
-
-		if data, err := value2Bytes(&rawValue); err == nil {
-			result[key] = data
-		} else {
-			return nil, err // !nashtsai! REVIEW, should return err or just error log?
-		}
-	}
-	return result, nil
-}
-
-func row2mapStr(rows *core.Rows, fields []string) (resultsMap map[string]string, err error) {
-	result := make(map[string]string)
-	scanResultContainers := make([]interface{}, len(fields))
-	for i := 0; i < len(fields); i++ {
-		var scanResultContainer interface{}
-		scanResultContainers[i] = &scanResultContainer
-	}
-	if err := rows.Scan(scanResultContainers...); err != nil {
-		return nil, err
-	}
-
-	for ii, key := range fields {
-		rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
-		//if row is null then ignore
-		if rawValue.Interface() == nil {
-			//fmt.Println("ignore ...", key, rawValue)
-			continue
-		}
-
-		if data, err := value2String(&rawValue); err == nil {
-			result[key] = data
-		} else {
-			return nil, err // !nashtsai! REVIEW, should return err or just error log?
-		}
-	}
-	return result, nil
-}
-
-func txQuery2(tx *core.Tx, sqlStr string, params ...interface{}) (resultsSlice []map[string]string, err error) {
-	rows, err := tx.Query(sqlStr, params...)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	return rows2Strings(rows)
-}
-
-func query2(db *core.DB, sqlStr string, params ...interface{}) (resultsSlice []map[string]string, err error) {
-	s, err := db.Prepare(sqlStr)
-	if err != nil {
-		return nil, err
-	}
-	defer s.Close()
-	rows, err := s.Query(params...)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	return rows2Strings(rows)
-}
-
 func setColumnInt(bean interface{}, col *core.Column, t int64) {
 	v, err := col.ValueOf(bean)
 	if err != nil {
@@ -514,7 +358,7 @@ func genCols(table *core.Table, session *Session, bean interface{}, useCol bool,
 
 	for _, col := range table.Columns() {
 		if useCol && !col.IsVersion && !col.IsCreated && !col.IsUpdated {
-			if _, ok := getFlagForColumn(session.Statement.columnMap, col); !ok {
+			if _, ok := getFlagForColumn(session.statement.columnMap, col); !ok {
 				continue
 			}
 		}
@@ -542,6 +386,10 @@ func genCols(table *core.Table, session *Session, bean interface{}, useCol bool,
 				if len(fieldValue.String()) == 0 {
 					continue
 				}
+			case reflect.Ptr:
+				if fieldValue.Pointer() == 0 {
+					continue
+				}
 			}
 		}
 
@@ -549,28 +397,32 @@ func genCols(table *core.Table, session *Session, bean interface{}, useCol bool,
 			continue
 		}
 
-		if session.Statement.ColumnStr != "" {
+		if session.statement.ColumnStr != "" {
-			if _, ok := getFlagForColumn(session.Statement.columnMap, col); !ok {
+			if _, ok := getFlagForColumn(session.statement.columnMap, col); !ok {
+				continue
+			} else if _, ok := session.statement.incrColumns[col.Name]; ok {
+				continue
+			} else if _, ok := session.statement.decrColumns[col.Name]; ok {
 				continue
 			}
 		}
-		if session.Statement.OmitStr != "" {
+		if session.statement.OmitStr != "" {
-			if _, ok := getFlagForColumn(session.Statement.columnMap, col); ok {
+			if _, ok := getFlagForColumn(session.statement.columnMap, col); ok {
 				continue
 			}
 		}
 
 		// !evalphobia! set fieldValue as nil when column is nullable and zero-value
-		if _, ok := getFlagForColumn(session.Statement.nullableMap, col); ok {
+		if _, ok := getFlagForColumn(session.statement.nullableMap, col); ok {
 			if col.Nullable && isZero(fieldValue.Interface()) {
 				var nilValue *int
 				fieldValue = reflect.ValueOf(nilValue)
 			}
 		}
 
-		if (col.IsCreated || col.IsUpdated) && session.Statement.UseAutoTime /*&& isZero(fieldValue.Interface())*/ {
+		if (col.IsCreated || col.IsUpdated) && session.statement.UseAutoTime /*&& isZero(fieldValue.Interface())*/ {
 			// if time is non-empty, then set to auto time
-			val, t := session.Engine.NowTime2(col.SQLType.Name)
+			val, t := session.engine.nowTime(col)
 			args = append(args, val)
 
 			var colName = col.Name
@@ -578,7 +430,7 @@ func genCols(table *core.Table, session *Session, bean interface{}, useCol bool,
 				col := table.GetColumn(colName)
 				setColumnTime(bean, col, t)
 			})
-		} else if col.IsVersion && session.Statement.checkVersion {
+		} else if col.IsVersion && session.statement.checkVersion {
 			args = append(args, 1)
 		} else {
 			arg, err := session.value2Interface(col, fieldValue)
@@ -589,7 +441,7 @@ func genCols(table *core.Table, session *Session, bean interface{}, useCol bool,
 		}
 
 		if includeQuote {
-			colNames = append(colNames, session.Engine.Quote(col.Name)+" = ?")
+			colNames = append(colNames, session.engine.Quote(col.Name)+" = ?")
 		} else {
 			colNames = append(colNames, col.Name)
 		}
@@ -602,7 +454,6 @@ func indexName(tableName, idxName string) string {
 }
 
 func getFlagForColumn(m map[string]bool, col *core.Column) (val bool, has bool) {
-
 	if len(m) == 0 {
 		return false, false
 	}

vendor/github.com/go-xorm/xorm/helpler_time.go

@@ -0,0 +1,21 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package xorm
+
+import "time"
+
+const (
+	zeroTime0 = "0000-00-00 00:00:00"
+	zeroTime1 = "0001-01-01 00:00:00"
+)
+
+func formatTime(t time.Time) string {
+	return t.Format("2006-01-02 15:04:05")
+}
+
+func isTimeZero(t time.Time) bool {
+	return t.IsZero() || formatTime(t) == zeroTime0 ||
+		formatTime(t) == zeroTime1
+}

vendor/github.com/go-xorm/xorm/interface.go

@@ -0,0 +1,103 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package xorm
+
+import (
+	"database/sql"
+	"reflect"
+	"time"
+
+	"github.com/go-xorm/core"
+)
+
+// Interface defines the interface which Engine, EngineGroup and Session will implementate.
+type Interface interface {
+	AllCols() *Session
+	Alias(alias string) *Session
+	Asc(colNames ...string) *Session
+	BufferSize(size int) *Session
+	Cols(columns ...string) *Session
+	Count(...interface{}) (int64, error)
+	CreateIndexes(bean interface{}) error
+	CreateUniques(bean interface{}) error
+	Decr(column string, arg ...interface{}) *Session
+	Desc(...string) *Session
+	Delete(interface{}) (int64, error)
+	Distinct(columns ...string) *Session
+	DropIndexes(bean interface{}) error
+	Exec(string, ...interface{}) (sql.Result, error)
+	Exist(bean ...interface{}) (bool, error)
+	Find(interface{}, ...interface{}) error
+	Get(interface{}) (bool, error)
+	GroupBy(keys string) *Session
+	ID(interface{}) *Session
+	In(string, ...interface{}) *Session
+	Incr(column string, arg ...interface{}) *Session
+	Insert(...interface{}) (int64, error)
+	InsertOne(interface{}) (int64, error)
+	IsTableEmpty(bean interface{}) (bool, error)
+	IsTableExist(beanOrTableName interface{}) (bool, error)
+	Iterate(interface{}, IterFunc) error
+	Limit(int, ...int) *Session
+	NoAutoCondition(...bool) *Session
+	NotIn(string, ...interface{}) *Session
+	Join(joinOperator string, tablename interface{}, condition string, args ...interface{}) *Session
+	Omit(columns ...string) *Session
+	OrderBy(order string) *Session
+	Ping() error
+	Query(sqlOrAgrs ...interface{}) (resultsSlice []map[string][]byte, err error)
+	QueryInterface(sqlorArgs ...interface{}) ([]map[string]interface{}, error)
+	QueryString(sqlorArgs ...interface{}) ([]map[string]string, error)
+	Rows(bean interface{}) (*Rows, error)
+	SetExpr(string, string) *Session
+	SQL(interface{}, ...interface{}) *Session
+	Sum(bean interface{}, colName string) (float64, error)
+	SumInt(bean interface{}, colName string) (int64, error)
+	Sums(bean interface{}, colNames ...string) ([]float64, error)
+	SumsInt(bean interface{}, colNames ...string) ([]int64, error)
+	Table(tableNameOrBean interface{}) *Session
+	Unscoped() *Session
+	Update(bean interface{}, condiBeans ...interface{}) (int64, error)
+	UseBool(...string) *Session
+	Where(interface{}, ...interface{}) *Session
+}
+
+// EngineInterface defines the interface which Engine, EngineGroup will implementate.
+type EngineInterface interface {
+	Interface
+
+	Before(func(interface{})) *Session
+	Charset(charset string) *Session
+	CreateTables(...interface{}) error
+	DBMetas() ([]*core.Table, error)
+	Dialect() core.Dialect
+	DropTables(...interface{}) error
+	DumpAllToFile(fp string, tp ...core.DbType) error
+	GetColumnMapper() core.IMapper
+	GetDefaultCacher() core.Cacher
+	GetTableMapper() core.IMapper
+	GetTZDatabase() *time.Location
+	GetTZLocation() *time.Location
+	NewSession() *Session
+	NoAutoTime() *Session
+	Quote(string) string
+	SetDefaultCacher(core.Cacher)
+	SetLogLevel(core.LogLevel)
+	SetMapper(core.IMapper)
+	SetTZDatabase(tz *time.Location)
+	SetTZLocation(tz *time.Location)
+	ShowSQL(show ...bool)
+	Sync(...interface{}) error
+	Sync2(...interface{}) error
+	StoreEngine(storeEngine string) *Session
+	TableInfo(bean interface{}) *Table
+	UnMapType(reflect.Type)
+}
+
+var (
+	_ Interface       = &Session{}
+	_ EngineInterface = &Engine{}
+	_ EngineInterface = &EngineGroup{}
+)

vendor/github.com/go-xorm/xorm/processors.go

@@ -29,13 +29,6 @@ type AfterSetProcessor interface {
 	AfterSet(string, Cell)
 }
 
-// !nashtsai! TODO enable BeforeValidateProcessor when xorm start to support validations
-//// Executed before an object is validated
-//type BeforeValidateProcessor interface {
-//    BeforeValidate()
-//}
-// --
-
 // AfterInsertProcessor executed after an object is persisted to the database
 type AfterInsertProcessor interface {
 	AfterInsert()
@@ -50,3 +43,36 @@ type AfterUpdateProcessor interface {
 type AfterDeleteProcessor interface {
 	AfterDelete()
 }
+
+// AfterLoadProcessor executed after an ojbect has been loaded from database
+type AfterLoadProcessor interface {
+	AfterLoad()
+}
+
+// AfterLoadSessionProcessor executed after an ojbect has been loaded from database with session parameter
+type AfterLoadSessionProcessor interface {
+	AfterLoad(*Session)
+}
+
+type executedProcessorFunc func(*Session, interface{}) error
+
+type executedProcessor struct {
+	fun     executedProcessorFunc
+	session *Session
+	bean    interface{}
+}
+
+func (executor *executedProcessor) execute() error {
+	return executor.fun(executor.session, executor.bean)
+}
+
+func (session *Session) executeProcessors() error {
+	processors := session.afterProcessors
+	session.afterProcessors = make([]executedProcessor, 0)
+	for _, processor := range processors {
+		if err := processor.execute(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/go-xorm/xorm/rows.go

@@ -17,7 +17,6 @@ type Rows struct {
 	NoTypeCheck bool
 
 	session   *Session
-	stmt      *core.Stmt
 	rows      *core.Rows
 	fields    []string
 	beanType  reflect.Type
@@ -29,50 +28,33 @@ func newRows(session *Session, bean interface{}) (*Rows, error) {
 	rows.session = session
 	rows.beanType = reflect.Indirect(reflect.ValueOf(bean)).Type()
 
-	defer rows.session.resetStatement()
-
 	var sqlStr string
 	var args []interface{}
+	var err error
 
-	rows.session.Statement.setRefValue(rValue(bean))
+	if err = rows.session.statement.setRefValue(rValue(bean)); err != nil {
-	if len(session.Statement.TableName()) <= 0 {
+		return nil, err
+	}
+
+	if len(session.statement.TableName()) <= 0 {
 		return nil, ErrTableNotFound
 	}
 
-	if rows.session.Statement.RawSQL == "" {
+	if rows.session.statement.RawSQL == "" {
-		sqlStr, args = rows.session.Statement.genGetSQL(bean)
+		sqlStr, args, err = rows.session.statement.genGetSQL(bean)
-	} else {
-		sqlStr = rows.session.Statement.RawSQL
-		args = rows.session.Statement.RawParams
-	}
-
-	for _, filter := range rows.session.Engine.dialect.Filters() {
-		sqlStr = filter.Do(sqlStr, session.Engine.dialect, rows.session.Statement.RefTable)
-	}
-
-	rows.session.saveLastSQL(sqlStr, args...)
-	var err error
-	if rows.session.prepareStmt {
-		rows.stmt, err = rows.session.DB().Prepare(sqlStr)
 		if err != nil {
-			rows.lastError = err
-			rows.Close()
-			return nil, err
-		}
-
-		rows.rows, err = rows.stmt.Query(args...)
-		if err != nil {
-			rows.lastError = err
-			rows.Close()
 			return nil, err
 		}
 	} else {
-		rows.rows, err = rows.session.DB().Query(sqlStr, args...)
+		sqlStr = rows.session.statement.RawSQL
-		if err != nil {
+		args = rows.session.statement.RawParams
-			rows.lastError = err
+	}
-			rows.Close()
+
-			return nil, err
+	rows.rows, err = rows.session.queryRows(sqlStr, args...)
-		}
+	if err != nil {
+		rows.lastError = err
+		rows.Close()
+		return nil, err
 	}
 
 	rows.fields, err = rows.rows.Columns()
@@ -113,15 +95,26 @@ func (rows *Rows) Scan(bean interface{}) error {
 	}
 
 	dataStruct := rValue(bean)
-	rows.session.Statement.setRefValue(dataStruct)
+	if err := rows.session.statement.setRefValue(dataStruct); err != nil {
-	_, err := rows.session.row2Bean(rows.rows, rows.fields, len(rows.fields), bean, &dataStruct, rows.session.Statement.RefTable)
+		return err
+	}
 
-	return err
+	scanResults, err := rows.session.row2Slice(rows.rows, rows.fields, bean)
+	if err != nil {
+		return err
+	}
+
+	_, err = rows.session.slice2Bean(scanResults, rows.fields, bean, &dataStruct, rows.session.statement.RefTable)
+	if err != nil {
+		return err
+	}
+
+	return rows.session.executeProcessors()
 }
 
 // Close session if session.IsAutoClose is true, and claimed any opened resources
 func (rows *Rows) Close() error {
-	if rows.session.IsAutoClose {
+	if rows.session.isAutoClose {
 		defer rows.session.Close()
 	}
 
@@ -129,17 +122,10 @@ func (rows *Rows) Close() error {
 		if rows.rows != nil {
 			rows.lastError = rows.rows.Close()
 			if rows.lastError != nil {
-				defer rows.stmt.Close()
 				return rows.lastError
 			}
 		}
-		if rows.stmt != nil {
-			rows.lastError = rows.stmt.Close()
-		}
 	} else {
-		if rows.stmt != nil {
-			defer rows.stmt.Close()
-		}
 		if rows.rows != nil {
 			defer rows.rows.Close()
 		}

vendor/github.com/go-xorm/xorm/session.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"hash/crc32"
 	"reflect"
-	"strconv"
 	"strings"
 	"time"
 
@@ -22,17 +21,16 @@ import (
 // kind of database operations.
 type Session struct {
 	db                     *core.DB
-	Engine                 *Engine
+	engine                 *Engine
-	Tx                     *core.Tx
+	tx                     *core.Tx
-	Statement              Statement
+	statement              Statement
-	IsAutoCommit           bool
+	isAutoCommit           bool
-	IsCommitedOrRollbacked bool
+	isCommitedOrRollbacked bool
-	TransType              string
+	isAutoClose            bool
-	IsAutoClose            bool
 
 	// Automatically reset the statement after operations that execute a SQL
 	// query such as Count(), Find(), Get(), ...
-	AutoResetStatement bool
+	autoResetStatement bool
 
 	// !nashtsai! storing these beans due to yet committed tx
 	afterInsertBeans map[interface{}]*[]func(interface{})
@@ -43,14 +41,17 @@ type Session struct {
 	beforeClosures []func(interface{})
 	afterClosures  []func(interface{})
 
+	afterProcessors []executedProcessor
+
 	prepareStmt bool
 	stmtCache   map[uint32]*core.Stmt //key: hash.Hash32 of (queryStr, len(queryStr))
-	cascadeDeep int
 
 	// !evalphobia! stored the last executed query on this session
 	//beforeSQLExec func(string, ...interface{})
 	lastSQL     string
 	lastSQLArgs []interface{}
+
+	err error
 }
 
 // Clone copy all the session's content and return a new session
@@ -61,12 +62,12 @@ func (session *Session) Clone() *Session {
 
 // Init reset the session as the init status.
 func (session *Session) Init() {
-	session.Statement.Init()
+	session.statement.Init()
-	session.Statement.Engine = session.Engine
+	session.statement.Engine = session.engine
-	session.IsAutoCommit = true
+	session.isAutoCommit = true
-	session.IsCommitedOrRollbacked = false
+	session.isCommitedOrRollbacked = false
-	session.IsAutoClose = false
+	session.isAutoClose = false
-	session.AutoResetStatement = true
+	session.autoResetStatement = true
 	session.prepareStmt = false
 
 	// !nashtsai! is lazy init better?
@@ -75,6 +76,9 @@ func (session *Session) Init() {
 	session.afterDeleteBeans = make(map[interface{}]*[]func(interface{}), 0)
 	session.beforeClosures = make([]func(interface{}), 0)
 	session.afterClosures = make([]func(interface{}), 0)
+	session.stmtCache = make(map[uint32]*core.Stmt)
+
+	session.afterProcessors = make([]executedProcessor, 0)
 
 	session.lastSQL = ""
 	session.lastSQLArgs = []interface{}{}
@@ -89,19 +93,23 @@ func (session *Session) Close() {
 	if session.db != nil {
 		// When Close be called, if session is a transaction and do not call
 		// Commit or Rollback, then call Rollback.
-		if session.Tx != nil && !session.IsCommitedOrRollbacked {
+		if session.tx != nil && !session.isCommitedOrRollbacked {
 			session.Rollback()
 		}
-		session.Tx = nil
+		session.tx = nil
 		session.stmtCache = nil
-		session.Init()
 		session.db = nil
 	}
 }
 
+// IsClosed returns if session is closed
+func (session *Session) IsClosed() bool {
+	return session.db == nil
+}
+
 func (session *Session) resetStatement() {
-	if session.AutoResetStatement {
+	if session.autoResetStatement {
-		session.Statement.Init()
+		session.statement.Init()
 	}
 }
 
@@ -129,75 +137,75 @@ func (session *Session) After(closures func(interface{})) *Session {
 
 // Table can input a string or pointer to struct for special a table to operate.
 func (session *Session) Table(tableNameOrBean interface{}) *Session {
-	session.Statement.Table(tableNameOrBean)
+	session.statement.Table(tableNameOrBean)
 	return session
 }
 
 // Alias set the table alias
 func (session *Session) Alias(alias string) *Session {
-	session.Statement.Alias(alias)
+	session.statement.Alias(alias)
 	return session
 }
 
 // NoCascade indicate that no cascade load child object
 func (session *Session) NoCascade() *Session {
-	session.Statement.UseCascade = false
+	session.statement.UseCascade = false
 	return session
 }
 
 // ForUpdate Set Read/Write locking for UPDATE
 func (session *Session) ForUpdate() *Session {
-	session.Statement.IsForUpdate = true
+	session.statement.IsForUpdate = true
 	return session
 }
 
 // NoAutoCondition disable generate SQL condition from beans
 func (session *Session) NoAutoCondition(no ...bool) *Session {
-	session.Statement.NoAutoCondition(no...)
+	session.statement.NoAutoCondition(no...)
 	return session
 }
 
 // Limit provide limit and offset query condition
 func (session *Session) Limit(limit int, start ...int) *Session {
-	session.Statement.Limit(limit, start...)
+	session.statement.Limit(limit, start...)
 	return session
 }
 
 // OrderBy provide order by query condition, the input parameter is the content
 // after order by on a sql statement.
 func (session *Session) OrderBy(order string) *Session {
-	session.Statement.OrderBy(order)
+	session.statement.OrderBy(order)
 	return session
 }
 
 // Desc provide desc order by query condition, the input parameters are columns.
 func (session *Session) Desc(colNames ...string) *Session {
-	session.Statement.Desc(colNames...)
+	session.statement.Desc(colNames...)
 	return session
 }
 
 // Asc provide asc order by query condition, the input parameters are columns.
 func (session *Session) Asc(colNames ...string) *Session {
-	session.Statement.Asc(colNames...)
+	session.statement.Asc(colNames...)
 	return session
 }
 
 // StoreEngine is only avialble mysql dialect currently
 func (session *Session) StoreEngine(storeEngine string) *Session {
-	session.Statement.StoreEngine = storeEngine
+	session.statement.StoreEngine = storeEngine
 	return session
 }
 
 // Charset is only avialble mysql dialect currently
 func (session *Session) Charset(charset string) *Session {
-	session.Statement.Charset = charset
+	session.statement.Charset = charset
 	return session
 }
 
 // Cascade indicates if loading sub Struct
 func (session *Session) Cascade(trueOrFalse ...bool) *Session {
 	if len(trueOrFalse) >= 1 {
-		session.Statement.UseCascade = trueOrFalse[0]
+		session.statement.UseCascade = trueOrFalse[0]
 	}
 	return session
 }
@@ -205,32 +213,32 @@ func (session *Session) Cascade(trueOrFalse ...bool) *Session {
 // NoCache ask this session do not retrieve data from cache system and
 // get data from database directly.
 func (session *Session) NoCache() *Session {
-	session.Statement.UseCache = false
+	session.statement.UseCache = false
 	return session
 }
 
 // Join join_operator should be one of INNER, LEFT OUTER, CROSS etc - this will be prepended to JOIN
 func (session *Session) Join(joinOperator string, tablename interface{}, condition string, args ...interface{}) *Session {
-	session.Statement.Join(joinOperator, tablename, condition, args...)
+	session.statement.Join(joinOperator, tablename, condition, args...)
 	return session
 }
 
 // GroupBy Generate Group By statement
 func (session *Session) GroupBy(keys string) *Session {
-	session.Statement.GroupBy(keys)
+	session.statement.GroupBy(keys)
 	return session
 }
 
 // Having Generate Having statement
 func (session *Session) Having(conditions string) *Session {
-	session.Statement.Having(conditions)
+	session.statement.Having(conditions)
 	return session
 }
 
 // DB db return the wrapper of sql.DB
 func (session *Session) DB() *core.DB {
 	if session.db == nil {
-		session.db = session.Engine.db
+		session.db = session.engine.db
 		session.stmtCache = make(map[uint32]*core.Stmt, 0)
 	}
 	return session.db
@@ -243,25 +251,25 @@ func cleanupProcessorsClosures(slices *[]func(interface{})) {
 }
 
 func (session *Session) canCache() bool {
-	if session.Statement.RefTable == nil ||
+	if session.statement.RefTable == nil ||
-		session.Statement.JoinStr != "" ||
+		session.statement.JoinStr != "" ||
-		session.Statement.RawSQL != "" ||
+		session.statement.RawSQL != "" ||
-		!session.Statement.UseCache ||
+		!session.statement.UseCache ||
-		session.Statement.IsForUpdate ||
+		session.statement.IsForUpdate ||
-		session.Tx != nil ||
+		session.tx != nil ||
-		len(session.Statement.selectStr) > 0 {
+		len(session.statement.selectStr) > 0 {
 		return false
 	}
 	return true
 }
 
-func (session *Session) doPrepare(sqlStr string) (stmt *core.Stmt, err error) {
+func (session *Session) doPrepare(db *core.DB, sqlStr string) (stmt *core.Stmt, err error) {
 	crc := crc32.ChecksumIEEE([]byte(sqlStr))
 	// TODO try hash(sqlStr+len(sqlStr))
 	var has bool
 	stmt, has = session.stmtCache[crc]
 	if !has {
-		stmt, err = session.DB().Prepare(sqlStr)
+		stmt, err = db.Prepare(sqlStr)
 		if err != nil {
 			return nil, err
 		}
@@ -273,18 +281,18 @@ func (session *Session) doPrepare(sqlStr string) (stmt *core.Stmt, err error) {
 func (session *Session) getField(dataStruct *reflect.Value, key string, table *core.Table, idx int) *reflect.Value {
 	var col *core.Column
 	if col = table.GetColumnIdx(key, idx); col == nil {
-		//session.Engine.logger.Warnf("table %v has no column %v. %v", table.Name, key, table.ColumnsSeq())
+		//session.engine.logger.Warnf("table %v has no column %v. %v", table.Name, key, table.ColumnsSeq())
 		return nil
 	}
 
 	fieldValue, err := col.ValueOfV(dataStruct)
 	if err != nil {
-		session.Engine.logger.Error(err)
+		session.engine.logger.Error(err)
 		return nil
 	}
 
 	if !fieldValue.IsValid() || !fieldValue.CanSet() {
-		session.Engine.logger.Warnf("table %v's column %v is not valid or cannot set", table.Name, key)
+		session.engine.logger.Warnf("table %v's column %v is not valid or cannot set", table.Name, key)
 		return nil
 	}
 	return fieldValue
@@ -293,28 +301,40 @@ func (session *Session) getField(dataStruct *reflect.Value, key string, table *c
 // Cell cell is a result of one column field
 type Cell *interface{}
 
-func (session *Session) rows2Beans(rows *core.Rows, fields []string, fieldsCount int,
+func (session *Session) rows2Beans(rows *core.Rows, fields []string,
 	table *core.Table, newElemFunc func([]string) reflect.Value,
 	sliceValueSetFunc func(*reflect.Value, core.PK) error) error {
 	for rows.Next() {
 		var newValue = newElemFunc(fields)
 		bean := newValue.Interface()
-		dataStruct := rValue(bean)
+		dataStruct := newValue.Elem()
-		pk, err := session.row2Bean(rows, fields, fieldsCount, bean, &dataStruct, table)
-		if err != nil {
-			return err
-		}
 
-		err = sliceValueSetFunc(&newValue, pk)
+		// handle beforeClosures
+		scanResults, err := session.row2Slice(rows, fields, bean)
 		if err != nil {
 			return err
 		}
+		pk, err := session.slice2Bean(scanResults, fields, bean, &dataStruct, table)
+		if err != nil {
+			return err
+		}
+		session.afterProcessors = append(session.afterProcessors, executedProcessor{
+			fun: func(*Session, interface{}) error {
+				return sliceValueSetFunc(&newValue, pk)
+			},
+			session: session,
+			bean:    bean,
+		})
 	}
 	return nil
 }
 
-func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount int, bean interface{}, dataStruct *reflect.Value, table *core.Table) (core.PK, error) {
+func (session *Session) row2Slice(rows *core.Rows, fields []string, bean interface{}) ([]interface{}, error) {
-	scanResults := make([]interface{}, fieldsCount)
+	for _, closure := range session.beforeClosures {
+		closure(bean)
+	}
+
+	scanResults := make([]interface{}, len(fields))
 	for i := 0; i < len(fields); i++ {
 		var cell interface{}
 		scanResults[i] = &cell
@@ -328,7 +348,10 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 			b.BeforeSet(key, Cell(scanResults[ii].(*interface{})))
 		}
 	}
+	return scanResults, nil
+}
 
+func (session *Session) slice2Bean(scanResults []interface{}, fields []string, bean interface{}, dataStruct *reflect.Value, table *core.Table) (core.PK, error) {
 	defer func() {
 		if b, hasAfterSet := bean.(AfterSetProcessor); hasAfterSet {
 			for ii, key := range fields {
@@ -337,6 +360,40 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 		}
 	}()
 
+	// handle afterClosures
+	for _, closure := range session.afterClosures {
+		session.afterProcessors = append(session.afterProcessors, executedProcessor{
+			fun: func(sess *Session, bean interface{}) error {
+				closure(bean)
+				return nil
+			},
+			session: session,
+			bean:    bean,
+		})
+	}
+
+	if a, has := bean.(AfterLoadProcessor); has {
+		session.afterProcessors = append(session.afterProcessors, executedProcessor{
+			fun: func(sess *Session, bean interface{}) error {
+				a.AfterLoad()
+				return nil
+			},
+			session: session,
+			bean:    bean,
+		})
+	}
+
+	if a, has := bean.(AfterLoadSessionProcessor); has {
+		session.afterProcessors = append(session.afterProcessors, executedProcessor{
+			fun: func(sess *Session, bean interface{}) error {
+				a.AfterLoad(sess)
+				return nil
+			},
+			session: session,
+			bean:    bean,
+		})
+	}
+
 	var tempMap = make(map[string]int)
 	var pk core.PK
 	for ii, key := range fields {
@@ -361,9 +418,11 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 			if fieldValue.CanAddr() {
 				if structConvert, ok := fieldValue.Addr().Interface().(core.Conversion); ok {
 					if data, err := value2Bytes(&rawValue); err == nil {
-						structConvert.FromDB(data)
+						if err := structConvert.FromDB(data); err != nil {
+							return nil, err
+						}
 					} else {
-						session.Engine.logger.Error(err)
+						return nil, err
 					}
 					continue
 				}
@@ -376,7 +435,7 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 					}
 					fieldValue.Interface().(core.Conversion).FromDB(data)
 				} else {
-					session.Engine.logger.Error(err)
+					return nil, err
 				}
 				continue
 			}
@@ -403,17 +462,19 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 				hasAssigned = true
 
 				if len(bs) > 0 {
+					if fieldType.Kind() == reflect.String {
+						fieldValue.SetString(string(bs))
+						continue
+					}
 					if fieldValue.CanAddr() {
 						err := json.Unmarshal(bs, fieldValue.Addr().Interface())
 						if err != nil {
-							session.Engine.logger.Error(key, err)
 							return nil, err
 						}
 					} else {
 						x := reflect.New(fieldType)
 						err := json.Unmarshal(bs, x.Interface())
 						if err != nil {
-							session.Engine.logger.Error(key, err)
 							return nil, err
 						}
 						fieldValue.Set(x.Elem())
@@ -438,14 +499,12 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 					if fieldValue.CanAddr() {
 						err := json.Unmarshal(bs, fieldValue.Addr().Interface())
 						if err != nil {
-							session.Engine.logger.Error(err)
 							return nil, err
 						}
 					} else {
 						x := reflect.New(fieldType)
 						err := json.Unmarshal(bs, x.Interface())
 						if err != nil {
-							session.Engine.logger.Error(err)
 							return nil, err
 						}
 						fieldValue.Set(x.Elem())
@@ -462,14 +521,19 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 								x := reflect.New(fieldType)
 								err := json.Unmarshal(vv.Bytes(), x.Interface())
 								if err != nil {
-									session.Engine.logger.Error(err)
 									return nil, err
 								}
 								fieldValue.Set(x.Elem())
 							} else {
-								for i := 0; i < fieldValue.Len(); i++ {
+								if fieldValue.Len() > 0 {
-									if i < vv.Len() {
+									for i := 0; i < fieldValue.Len(); i++ {
-										fieldValue.Index(i).Set(vv.Index(i))
+										if i < vv.Len() {
+											fieldValue.Index(i).Set(vv.Index(i))
+										}
+									}
+								} else {
+									for i := 0; i < vv.Len(); i++ {
+										fieldValue.Set(reflect.Append(*fieldValue, vv.Index(i)))
 									}
 								}
 							}
@@ -509,57 +573,38 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 				}
 			case reflect.Struct:
 				if fieldType.ConvertibleTo(core.TimeType) {
+					dbTZ := session.engine.DatabaseTZ
+					if col.TimeZone != nil {
+						dbTZ = col.TimeZone
+					}
+
 					if rawValueType == core.TimeType {
 						hasAssigned = true
 
 						t := vv.Convert(core.TimeType).Interface().(time.Time)
 
 						z, _ := t.Zone()
-						dbTZ := session.Engine.DatabaseTZ
-						if dbTZ == nil {
-							if session.Engine.dialect.DBType() == core.SQLITE {
-								dbTZ = time.UTC
-							} else {
-								dbTZ = time.Local
-							}
-						}
-
 						// set new location if database don't save timezone or give an incorrect timezone
 						if len(z) == 0 || t.Year() == 0 || t.Location().String() != dbTZ.String() { // !nashtsai! HACK tmp work around for lib/pq doesn't properly time with location
-							session.Engine.logger.Debugf("empty zone key[%v] : %v | zone: %v | location: %+v\n", key, t, z, *t.Location())
+							session.engine.logger.Debugf("empty zone key[%v] : %v | zone: %v | location: %+v\n", key, t, z, *t.Location())
 							t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(),
 								t.Minute(), t.Second(), t.Nanosecond(), dbTZ)
 						}
 
-						// !nashtsai! convert to engine location
+						t = t.In(session.engine.TZLocation)
-						if col.TimeZone == nil {
-							t = t.In(session.Engine.TZLocation)
-						} else {
-							t = t.In(col.TimeZone)
-						}
 						fieldValue.Set(reflect.ValueOf(t).Convert(fieldType))
-
-						// t = fieldValue.Interface().(time.Time)
-						// z, _ = t.Zone()
-						// session.Engine.LogDebug("fieldValue key[%v]: %v | zone: %v | location: %+v\n", key, t, z, *t.Location())
 					} else if rawValueType == core.IntType || rawValueType == core.Int64Type ||
 						rawValueType == core.Int32Type {
 						hasAssigned = true
-						var tz *time.Location
+
-						if col.TimeZone == nil {
+						t := time.Unix(vv.Int(), 0).In(session.engine.TZLocation)
-							tz = session.Engine.TZLocation
-						} else {
-							tz = col.TimeZone
-						}
-						t := time.Unix(vv.Int(), 0).In(tz)
-						//vv = reflect.ValueOf(t)
 						fieldValue.Set(reflect.ValueOf(t).Convert(fieldType))
 					} else {
 						if d, ok := vv.Interface().([]uint8); ok {
 							hasAssigned = true
 							t, err := session.byte2Time(col, d)
 							if err != nil {
-								session.Engine.logger.Error("byte2Time error:", err.Error())
+								session.engine.logger.Error("byte2Time error:", err.Error())
 								hasAssigned = false
 							} else {
 								fieldValue.Set(reflect.ValueOf(t).Convert(fieldType))
@@ -568,20 +613,20 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 							hasAssigned = true
 							t, err := session.str2Time(col, d)
 							if err != nil {
-								session.Engine.logger.Error("byte2Time error:", err.Error())
+								session.engine.logger.Error("byte2Time error:", err.Error())
 								hasAssigned = false
 							} else {
 								fieldValue.Set(reflect.ValueOf(t).Convert(fieldType))
 							}
 						} else {
-							panic(fmt.Sprintf("rawValueType is %v, value is %v", rawValueType, vv.Interface()))
+							return nil, fmt.Errorf("rawValueType is %v, value is %v", rawValueType, vv.Interface())
 						}
 					}
 				} else if nulVal, ok := fieldValue.Addr().Interface().(sql.Scanner); ok {
 					// !<winxxp>! 增加支持sql.Scanner接口的结构，如sql.NullString
 					hasAssigned = true
 					if err := nulVal.Scan(vv.Interface()); err != nil {
-						session.Engine.logger.Error("sql.Sanner error:", err.Error())
+						session.engine.logger.Error("sql.Sanner error:", err.Error())
 						hasAssigned = false
 					}
 				} else if col.SQLType.IsJson() {
@@ -591,7 +636,6 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 						if len([]byte(vv.String())) > 0 {
 							err := json.Unmarshal([]byte(vv.String()), x.Interface())
 							if err != nil {
-								session.Engine.logger.Error(err)
 								return nil, err
 							}
 							fieldValue.Set(x.Elem())
@@ -602,76 +646,45 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 						if len(vv.Bytes()) > 0 {
 							err := json.Unmarshal(vv.Bytes(), x.Interface())
 							if err != nil {
-								session.Engine.logger.Error(err)
 								return nil, err
 							}
 							fieldValue.Set(x.Elem())
 						}
 					}
-				} else if session.Statement.UseCascade {
+				} else if session.statement.UseCascade {
-					table := session.Engine.autoMapType(*fieldValue)
+					table, err := session.engine.autoMapType(*fieldValue)
-					if table != nil {
+					if err != nil {
-						hasAssigned = true
+						return nil, err
-						if len(table.PrimaryKeys) != 1 {
+					}
-							panic("unsupported non or composited primary key cascade")
-						}
-						var pk = make(core.PK, len(table.PrimaryKeys))
 
-						switch rawValueType.Kind() {
+					hasAssigned = true
-						case reflect.Int64:
+					if len(table.PrimaryKeys) != 1 {
-							pk[0] = vv.Int()
+						return nil, errors.New("unsupported non or composited primary key cascade")
-						case reflect.Int:
+					}
-							pk[0] = int(vv.Int())
+					var pk = make(core.PK, len(table.PrimaryKeys))
-						case reflect.Int32:
+					pk[0], err = asKind(vv, rawValueType)
-							pk[0] = int32(vv.Int())
+					if err != nil {
-						case reflect.Int16:
+						return nil, err
-							pk[0] = int16(vv.Int())
+					}
-						case reflect.Int8:
-							pk[0] = int8(vv.Int())
-						case reflect.Uint64:
-							pk[0] = vv.Uint()
-						case reflect.Uint:
-							pk[0] = uint(vv.Uint())
-						case reflect.Uint32:
-							pk[0] = uint32(vv.Uint())
-						case reflect.Uint16:
-							pk[0] = uint16(vv.Uint())
-						case reflect.Uint8:
-							pk[0] = uint8(vv.Uint())
-						case reflect.String:
-							pk[0] = vv.String()
-						case reflect.Slice:
-							pk[0], _ = strconv.ParseInt(string(rawValue.Interface().([]byte)), 10, 64)
-						default:
-							panic(fmt.Sprintf("unsupported primary key type: %v, %v", rawValueType, fieldValue))
-						}
 
-						if !isPKZero(pk) {
+					if !isPKZero(pk) {
-							// !nashtsai! TODO for hasOne relationship, it's preferred to use join query for eager fetch
+						// !nashtsai! TODO for hasOne relationship, it's preferred to use join query for eager fetch
-							// however, also need to consider adding a 'lazy' attribute to xorm tag which allow hasOne
+						// however, also need to consider adding a 'lazy' attribute to xorm tag which allow hasOne
-							// property to be fetched lazily
+						// property to be fetched lazily
-							structInter := reflect.New(fieldValue.Type())
+						structInter := reflect.New(fieldValue.Type())
-							newsession := session.Engine.NewSession()
+						has, err := session.ID(pk).NoCascade().get(structInter.Interface())
-							defer newsession.Close()
+						if err != nil {
-							has, err := newsession.Id(pk).NoCascade().Get(structInter.Interface())
+							return nil, err
-							if err != nil {
+						}
-								return nil, err
+						if has {
-							}
+							fieldValue.Set(structInter.Elem())
-							if has {
+						} else {
-								//v := structInter.Elem().Interface()
+							return nil, errors.New("cascade obj is not exist")
-								//fieldValue.Set(reflect.ValueOf(v))
-								fieldValue.Set(structInter.Elem())
-							} else {
-								return nil, errors.New("cascade obj is not exist")
-							}
 						}
-					} else {
-						session.Engine.logger.Error("unsupported struct type in Scan: ", fieldValue.Type().String())
 					}
 				}
 			case reflect.Ptr:
 				// !nashtsai! TODO merge duplicated codes above
-				//typeStr := fieldType.String()
 				switch fieldType {
 				// following types case matching ptr's native type, therefore assign ptr directly
 				case core.PtrStringType:
@@ -769,10 +782,9 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 					if len([]byte(vv.String())) > 0 {
 						err := json.Unmarshal([]byte(vv.String()), &x)
 						if err != nil {
-							session.Engine.logger.Error(err)
+							return nil, err
-						} else {
-							fieldValue.Set(reflect.ValueOf(&x))
 						}
+						fieldValue.Set(reflect.ValueOf(&x))
 					}
 					hasAssigned = true
 				case core.Complex128Type:
@@ -780,24 +792,23 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 					if len([]byte(vv.String())) > 0 {
 						err := json.Unmarshal([]byte(vv.String()), &x)
 						if err != nil {
-							session.Engine.logger.Error(err)
+							return nil, err
-						} else {
-							fieldValue.Set(reflect.ValueOf(&x))
 						}
+						fieldValue.Set(reflect.ValueOf(&x))
 					}
 					hasAssigned = true
 				} // switch fieldType
-				// default:
-				// 	session.Engine.LogError("unsupported type in Scan: ", reflect.TypeOf(v).String())
 			} // switch fieldType.Kind()
 
 			// !nashtsai! for value can't be assigned directly fallback to convert to []byte then back to value
 			if !hasAssigned {
 				data, err := value2Bytes(&rawValue)
-				if err == nil {
+				if err != nil {
-					session.bytes2Value(col, fieldValue, data)
+					return nil, err
-				} else {
+				}
-					session.Engine.logger.Error(err.Error())
+
+				if err = session.bytes2Value(col, fieldValue, data); err != nil {
+					return nil, err
 				}
 			}
 		}
@@ -805,19 +816,11 @@ func (session *Session) row2Bean(rows *core.Rows, fields []string, fieldsCount i
 	return pk, nil
 }
 
-func (session *Session) queryPreprocess(sqlStr *string, paramStr ...interface{}) {
-	for _, filter := range session.Engine.dialect.Filters() {
-		*sqlStr = filter.Do(*sqlStr, session.Engine.dialect, session.Statement.RefTable)
-	}
-
-	session.saveLastSQL(*sqlStr, paramStr...)
-}
-
 // saveLastSQL stores executed query information
 func (session *Session) saveLastSQL(sql string, args ...interface{}) {
 	session.lastSQL = sql
 	session.lastSQLArgs = args
-	session.Engine.logSQL(sql, args...)
+	session.engine.logSQL(sql, args...)
 }
 
 // LastSQL returns last query information
@@ -827,8 +830,8 @@ func (session *Session) LastSQL() (string, []interface{}) {
 
 // tbName get some table's table name
 func (session *Session) tbNameNoSchema(table *core.Table) string {
-	if len(session.Statement.AltTableName) > 0 {
+	if len(session.statement.AltTableName) > 0 {
-		return session.Statement.AltTableName
+		return session.statement.AltTableName
 	}
 
 	return table.Name
@@ -836,6 +839,6 @@ func (session *Session) tbNameNoSchema(table *core.Table) string {
 
 // Unscoped always disable struct tag "deleted"
 func (session *Session) Unscoped() *Session {
-	session.Statement.Unscoped()
+	session.statement.Unscoped()
 	return session
 }

vendor/github.com/go-xorm/xorm/session_cols.go

@@ -6,43 +6,43 @@ package xorm
 
 // Incr provides a query string like "count = count + 1"
 func (session *Session) Incr(column string, arg ...interface{}) *Session {
-	session.Statement.Incr(column, arg...)
+	session.statement.Incr(column, arg...)
 	return session
 }
 
 // Decr provides a query string like "count = count - 1"
 func (session *Session) Decr(column string, arg ...interface{}) *Session {
-	session.Statement.Decr(column, arg...)
+	session.statement.Decr(column, arg...)
 	return session
 }
 
 // SetExpr provides a query string like "column = {expression}"
 func (session *Session) SetExpr(column string, expression string) *Session {
-	session.Statement.SetExpr(column, expression)
+	session.statement.SetExpr(column, expression)
 	return session
 }
 
 // Select provides some columns to special
 func (session *Session) Select(str string) *Session {
-	session.Statement.Select(str)
+	session.statement.Select(str)
 	return session
 }
 
 // Cols provides some columns to special
 func (session *Session) Cols(columns ...string) *Session {
-	session.Statement.Cols(columns...)
+	session.statement.Cols(columns...)
 	return session
 }
 
 // AllCols ask all columns
 func (session *Session) AllCols() *Session {
-	session.Statement.AllCols()
+	session.statement.AllCols()
 	return session
 }
 
 // MustCols specify some columns must use even if they are empty
 func (session *Session) MustCols(columns ...string) *Session {
-	session.Statement.MustCols(columns...)
+	session.statement.MustCols(columns...)
 	return session
 }
 
@@ -52,7 +52,7 @@ func (session *Session) MustCols(columns ...string) *Session {
 // If no parameters, it will use all the bool field of struct, or
 // it will use parameters's columns
 func (session *Session) UseBool(columns ...string) *Session {
-	session.Statement.UseBool(columns...)
+	session.statement.UseBool(columns...)
 	return session
 }
 
@@ -60,25 +60,25 @@ func (session *Session) UseBool(columns ...string) *Session {
 // distinct will not be cached because cache system need id,
 // but distinct will not provide id
 func (session *Session) Distinct(columns ...string) *Session {
-	session.Statement.Distinct(columns...)
+	session.statement.Distinct(columns...)
 	return session
 }
 
 // Omit Only not use the parameters as select or update columns
 func (session *Session) Omit(columns ...string) *Session {
-	session.Statement.Omit(columns...)
+	session.statement.Omit(columns...)
 	return session
 }
 
 // Nullable Set null when column is zero-value and nullable for update
 func (session *Session) Nullable(columns ...string) *Session {
-	session.Statement.Nullable(columns...)
+	session.statement.Nullable(columns...)
 	return session
 }
 
 // NoAutoTime means do not automatically give created field and updated field
 // the current time on the current session temporarily
 func (session *Session) NoAutoTime() *Session {
-	session.Statement.UseAutoTime = false
+	session.statement.UseAutoTime = false
 	return session
 }

vendor/github.com/go-xorm/xorm/session_cond.go

@@ -17,25 +17,25 @@ func (session *Session) Sql(query string, args ...interface{}) *Session {
 // SQL provides raw sql input parameter. When you have a complex SQL statement
 // and cannot use Where, Id, In and etc. Methods to describe, you can use SQL.
 func (session *Session) SQL(query interface{}, args ...interface{}) *Session {
-	session.Statement.SQL(query, args...)
+	session.statement.SQL(query, args...)
 	return session
 }
 
 // Where provides custom query condition.
 func (session *Session) Where(query interface{}, args ...interface{}) *Session {
-	session.Statement.Where(query, args...)
+	session.statement.Where(query, args...)
 	return session
 }
 
 // And provides custom query condition.
 func (session *Session) And(query interface{}, args ...interface{}) *Session {
-	session.Statement.And(query, args...)
+	session.statement.And(query, args...)
 	return session
 }
 
 // Or provides custom query condition.
 func (session *Session) Or(query interface{}, args ...interface{}) *Session {
-	session.Statement.Or(query, args...)
+	session.statement.Or(query, args...)
 	return session
 }
 
@@ -48,23 +48,23 @@ func (session *Session) Id(id interface{}) *Session {
 
 // ID provides converting id as a query condition
 func (session *Session) ID(id interface{}) *Session {
-	session.Statement.ID(id)
+	session.statement.ID(id)
 	return session
 }
 
 // In provides a query string like "id in (1, 2, 3)"
 func (session *Session) In(column string, args ...interface{}) *Session {
-	session.Statement.In(column, args...)
+	session.statement.In(column, args...)
 	return session
 }
 
 // NotIn provides a query string like "id in (1, 2, 3)"
 func (session *Session) NotIn(column string, args ...interface{}) *Session {
-	session.Statement.NotIn(column, args...)
+	session.statement.NotIn(column, args...)
 	return session
 }
 
-// Conds returns session query conditions
+// Conds returns session query conditions except auto bean conditions
 func (session *Session) Conds() builder.Cond {
-	return session.Statement.cond
+	return session.statement.cond
 }

vendor/github.com/go-xorm/xorm/session_convert.go

@@ -23,41 +23,38 @@ func (session *Session) str2Time(col *core.Column, data string) (outTime time.Ti
 	var x time.Time
 	var err error
 
-	if sdata == "0000-00-00 00:00:00" ||
+	var parseLoc = session.engine.DatabaseTZ
-		sdata == "0001-01-01 00:00:00" {
+	if col.TimeZone != nil {
+		parseLoc = col.TimeZone
+	}
+
+	if sdata == zeroTime0 || sdata == zeroTime1 {
 	} else if !strings.ContainsAny(sdata, "- :") { // !nashtsai! has only found that mymysql driver is using this for time type column
 		// time stamp
 		sd, err := strconv.ParseInt(sdata, 10, 64)
 		if err == nil {
 			x = time.Unix(sd, 0)
-			// !nashtsai! HACK mymysql driver is causing Local location being change to CHAT and cause wrong time conversion
+			//session.engine.logger.Debugf("time(0) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
-			if col.TimeZone == nil {
-				x = x.In(session.Engine.TZLocation)
-			} else {
-				x = x.In(col.TimeZone)
-			}
-			session.Engine.logger.Debugf("time(0) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
 		} else {
-			session.Engine.logger.Debugf("time(0) err key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
+			//session.engine.logger.Debugf("time(0) err key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
 		}
 	} else if len(sdata) > 19 && strings.Contains(sdata, "-") {
-		x, err = time.ParseInLocation(time.RFC3339Nano, sdata, session.Engine.TZLocation)
+		x, err = time.ParseInLocation(time.RFC3339Nano, sdata, parseLoc)
-		session.Engine.logger.Debugf("time(1) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
+		session.engine.logger.Debugf("time(1) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
 		if err != nil {
-			x, err = time.ParseInLocation("2006-01-02 15:04:05.999999999", sdata, session.Engine.TZLocation)
+			x, err = time.ParseInLocation("2006-01-02 15:04:05.999999999", sdata, parseLoc)
-			session.Engine.logger.Debugf("time(2) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
+			//session.engine.logger.Debugf("time(2) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
 		}
 		if err != nil {
-			x, err = time.ParseInLocation("2006-01-02 15:04:05.9999999 Z07:00", sdata, session.Engine.TZLocation)
+			x, err = time.ParseInLocation("2006-01-02 15:04:05.9999999 Z07:00", sdata, parseLoc)
-			session.Engine.logger.Debugf("time(3) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
+			//session.engine.logger.Debugf("time(3) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
 		}
-
 	} else if len(sdata) == 19 && strings.Contains(sdata, "-") {
-		x, err = time.ParseInLocation("2006-01-02 15:04:05", sdata, session.Engine.TZLocation)
+		x, err = time.ParseInLocation("2006-01-02 15:04:05", sdata, parseLoc)
-		session.Engine.logger.Debugf("time(4) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
+		//session.engine.logger.Debugf("time(4) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
 	} else if len(sdata) == 10 && sdata[4] == '-' && sdata[7] == '-' {
-		x, err = time.ParseInLocation("2006-01-02", sdata, session.Engine.TZLocation)
+		x, err = time.ParseInLocation("2006-01-02", sdata, parseLoc)
-		session.Engine.logger.Debugf("time(5) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
+		//session.engine.logger.Debugf("time(5) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
 	} else if col.SQLType.Name == core.Time {
 		if strings.Contains(sdata, " ") {
 			ssd := strings.Split(sdata, " ")
@@ -65,13 +62,13 @@ func (session *Session) str2Time(col *core.Column, data string) (outTime time.Ti
 		}
 
 		sdata = strings.TrimSpace(sdata)
-		if session.Engine.dialect.DBType() == core.MYSQL && len(sdata) > 8 {
+		if session.engine.dialect.DBType() == core.MYSQL && len(sdata) > 8 {
 			sdata = sdata[len(sdata)-8:]
 		}
 
 		st := fmt.Sprintf("2006-01-02 %v", sdata)
-		x, err = time.ParseInLocation("2006-01-02 15:04:05", st, session.Engine.TZLocation)
+		x, err = time.ParseInLocation("2006-01-02 15:04:05", st, parseLoc)
-		session.Engine.logger.Debugf("time(6) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
+		//session.engine.logger.Debugf("time(6) key[%v]: %+v | sdata: [%v]\n", col.FieldName, x, sdata)
 	} else {
 		outErr = fmt.Errorf("unsupported time format %v", sdata)
 		return
@@ -80,7 +77,7 @@ func (session *Session) str2Time(col *core.Column, data string) (outTime time.Ti
 		outErr = fmt.Errorf("unsupported time format %v: %v", sdata, err)
 		return
 	}
-	outTime = x
+	outTime = x.In(session.engine.TZLocation)
 	return
 }
 
@@ -108,7 +105,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 		if len(data) > 0 {
 			err := json.Unmarshal(data, x.Interface())
 			if err != nil {
-				session.Engine.logger.Error(err)
+				session.engine.logger.Error(err)
 				return err
 			}
 			fieldValue.Set(x.Elem())
@@ -122,7 +119,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 			if len(data) > 0 {
 				err := json.Unmarshal(data, x.Interface())
 				if err != nil {
-					session.Engine.logger.Error(err)
+					session.engine.logger.Error(err)
 					return err
 				}
 				fieldValue.Set(x.Elem())
@@ -135,7 +132,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 				if len(data) > 0 {
 					err := json.Unmarshal(data, x.Interface())
 					if err != nil {
-						session.Engine.logger.Error(err)
+						session.engine.logger.Error(err)
 						return err
 					}
 					fieldValue.Set(x.Elem())
@@ -147,8 +144,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 	case reflect.String:
 		fieldValue.SetString(string(data))
 	case reflect.Bool:
-		d := string(data)
+		v, err := asBool(data)
-		v, err := strconv.ParseBool(d)
 		if err != nil {
 			return fmt.Errorf("arg %v as bool: %s", key, err.Error())
 		}
@@ -159,7 +155,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 		var err error
 		// for mysql, when use bit, it returned \x01
 		if col.SQLType.Name == core.Bit &&
-			session.Engine.dialect.DBType() == core.MYSQL { // !nashtsai! TODO dialect needs to provide conversion interface API
+			session.engine.dialect.DBType() == core.MYSQL { // !nashtsai! TODO dialect needs to provide conversion interface API
 			if len(data) == 1 {
 				x = int64(data[0])
 			} else {
@@ -207,41 +203,39 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 				}
 				v = x
 				fieldValue.Set(reflect.ValueOf(v).Convert(fieldType))
-			} else if session.Statement.UseCascade {
+			} else if session.statement.UseCascade {
-				table := session.Engine.autoMapType(*fieldValue)
+				table, err := session.engine.autoMapType(*fieldValue)
-				if table != nil {
+				if err != nil {
-					// TODO: current only support 1 primary key
+					return err
-					if len(table.PrimaryKeys) > 1 {
+				}
-						panic("unsupported composited primary key cascade")
+
-					}
+				// TODO: current only support 1 primary key
-					var pk = make(core.PK, len(table.PrimaryKeys))
+				if len(table.PrimaryKeys) > 1 {
-					rawValueType := table.ColumnType(table.PKColumns()[0].FieldName)
+					return errors.New("unsupported composited primary key cascade")
-					var err error
+				}
-					pk[0], err = str2PK(string(data), rawValueType)
+
+				var pk = make(core.PK, len(table.PrimaryKeys))
+				rawValueType := table.ColumnType(table.PKColumns()[0].FieldName)
+				pk[0], err = str2PK(string(data), rawValueType)
+				if err != nil {
+					return err
+				}
+
+				if !isPKZero(pk) {
+					// !nashtsai! TODO for hasOne relationship, it's preferred to use join query for eager fetch
+					// however, also need to consider adding a 'lazy' attribute to xorm tag which allow hasOne
+					// property to be fetched lazily
+					structInter := reflect.New(fieldValue.Type())
+					has, err := session.ID(pk).NoCascade().get(structInter.Interface())
 					if err != nil {
 						return err
 					}
-
+					if has {
-					if !isPKZero(pk) {
+						v = structInter.Elem().Interface()
-						// !nashtsai! TODO for hasOne relationship, it's preferred to use join query for eager fetch
+						fieldValue.Set(reflect.ValueOf(v))
-						// however, also need to consider adding a 'lazy' attribute to xorm tag which allow hasOne
+					} else {
-						// property to be fetched lazily
+						return errors.New("cascade obj is not exist")
-						structInter := reflect.New(fieldValue.Type())
-						newsession := session.Engine.NewSession()
-						defer newsession.Close()
-						has, err := newsession.Id(pk).NoCascade().Get(structInter.Interface())
-						if err != nil {
-							return err
-						}
-						if has {
-							v = structInter.Elem().Interface()
-							fieldValue.Set(reflect.ValueOf(v))
-						} else {
-							return errors.New("cascade obj is not exist")
-						}
 					}
-				} else {
-					return fmt.Errorf("unsupported struct type in Scan: %s", fieldValue.Type().String())
 				}
 			}
 		}
@@ -267,7 +261,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 			if len(data) > 0 {
 				err := json.Unmarshal(data, &x)
 				if err != nil {
-					session.Engine.logger.Error(err)
+					session.engine.logger.Error(err)
 					return err
 				}
 				fieldValue.Set(reflect.ValueOf(&x).Convert(fieldType))
@@ -278,7 +272,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 			if len(data) > 0 {
 				err := json.Unmarshal(data, &x)
 				if err != nil {
-					session.Engine.logger.Error(err)
+					session.engine.logger.Error(err)
 					return err
 				}
 				fieldValue.Set(reflect.ValueOf(&x).Convert(fieldType))
@@ -350,7 +344,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 			var err error
 			// for mysql, when use bit, it returned \x01
 			if col.SQLType.Name == core.Bit &&
-				strings.Contains(session.Engine.DriverName(), "mysql") {
+				strings.Contains(session.engine.DriverName(), "mysql") {
 				if len(data) == 1 {
 					x = int64(data[0])
 				} else {
@@ -375,7 +369,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 			var err error
 			// for mysql, when use bit, it returned \x01
 			if col.SQLType.Name == core.Bit &&
-				strings.Contains(session.Engine.DriverName(), "mysql") {
+				strings.Contains(session.engine.DriverName(), "mysql") {
 				if len(data) == 1 {
 					x = int(data[0])
 				} else {
@@ -403,7 +397,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 			var err error
 			// for mysql, when use bit, it returned \x01
 			if col.SQLType.Name == core.Bit &&
-				session.Engine.dialect.DBType() == core.MYSQL {
+				session.engine.dialect.DBType() == core.MYSQL {
 				if len(data) == 1 {
 					x = int32(data[0])
 				} else {
@@ -431,7 +425,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 			var err error
 			// for mysql, when use bit, it returned \x01
 			if col.SQLType.Name == core.Bit &&
-				strings.Contains(session.Engine.DriverName(), "mysql") {
+				strings.Contains(session.engine.DriverName(), "mysql") {
 				if len(data) == 1 {
 					x = int8(data[0])
 				} else {
@@ -459,7 +453,7 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 			var err error
 			// for mysql, when use bit, it returned \x01
 			if col.SQLType.Name == core.Bit &&
-				strings.Contains(session.Engine.DriverName(), "mysql") {
+				strings.Contains(session.engine.DriverName(), "mysql") {
 				if len(data) == 1 {
 					x = int16(data[0])
 				} else {
@@ -491,37 +485,37 @@ func (session *Session) bytes2Value(col *core.Column, fieldValue *reflect.Value,
 				v = x
 				fieldValue.Set(reflect.ValueOf(&x))
 			default:
-				if session.Statement.UseCascade {
+				if session.statement.UseCascade {
 					structInter := reflect.New(fieldType.Elem())
-					table := session.Engine.autoMapType(structInter.Elem())
+					table, err := session.engine.autoMapType(structInter.Elem())
-					if table != nil {
+					if err != nil {
-						if len(table.PrimaryKeys) > 1 {
+						return err
-							panic("unsupported composited primary key cascade")
+					}
-						}
+
-						var pk = make(core.PK, len(table.PrimaryKeys))
+					if len(table.PrimaryKeys) > 1 {
-						var err error
+						return errors.New("unsupported composited primary key cascade")
-						rawValueType := table.ColumnType(table.PKColumns()[0].FieldName)
+					}
-						pk[0], err = str2PK(string(data), rawValueType)
+
+					var pk = make(core.PK, len(table.PrimaryKeys))
+					rawValueType := table.ColumnType(table.PKColumns()[0].FieldName)
+					pk[0], err = str2PK(string(data), rawValueType)
+					if err != nil {
+						return err
+					}
+
+					if !isPKZero(pk) {
+						// !nashtsai! TODO for hasOne relationship, it's preferred to use join query for eager fetch
+						// however, also need to consider adding a 'lazy' attribute to xorm tag which allow hasOne
+						// property to be fetched lazily
+						has, err := session.ID(pk).NoCascade().get(structInter.Interface())
 						if err != nil {
 							return err
 						}
-
+						if has {
-						if !isPKZero(pk) {
+							v = structInter.Interface()
-							// !nashtsai! TODO for hasOne relationship, it's preferred to use join query for eager fetch
+							fieldValue.Set(reflect.ValueOf(v))
-							// however, also need to consider adding a 'lazy' attribute to xorm tag which allow hasOne
+						} else {
-							// property to be fetched lazily
+							return errors.New("cascade obj is not exist")
-							newsession := session.Engine.NewSession()
-							defer newsession.Close()
-							has, err := newsession.Id(pk).NoCascade().Get(structInter.Interface())
-							if err != nil {
-								return err
-							}
-							if has {
-								v = structInter.Interface()
-								fieldValue.Set(reflect.ValueOf(v))
-							} else {
-								return errors.New("cascade obj is not exist")
-							}
 						}
 					}
 				} else {
@@ -570,7 +564,7 @@ func (session *Session) value2Interface(col *core.Column, fieldValue reflect.Val
 		if fieldValue.IsNil() {
 			return nil, nil
 		} else if !fieldValue.IsValid() {
-			session.Engine.logger.Warn("the field[", col.FieldName, "] is invalid")
+			session.engine.logger.Warn("the field[", col.FieldName, "] is invalid")
 			return nil, nil
 		} else {
 			// !nashtsai! deference pointer type to instance type
@@ -588,12 +582,7 @@ func (session *Session) value2Interface(col *core.Column, fieldValue reflect.Val
 	case reflect.Struct:
 		if fieldType.ConvertibleTo(core.TimeType) {
 			t := fieldValue.Convert(core.TimeType).Interface().(time.Time)
-			if session.Engine.dialect.DBType() == core.MSSQL {
+			tf := session.engine.formatColTime(col, t)
-				if t.IsZero() {
-					return nil, nil
-				}
-			}
-			tf := session.Engine.FormatTime(col.SQLType.Name, t)
 			return tf, nil
 		}
 
@@ -603,7 +592,10 @@ func (session *Session) value2Interface(col *core.Column, fieldValue reflect.Val
 				return v.Value()
 			}
 
-			fieldTable := session.Engine.autoMapType(fieldValue)
+			fieldTable, err := session.engine.autoMapType(fieldValue)
+			if err != nil {
+				return nil, err
+			}
 			if len(fieldTable.PrimaryKeys) == 1 {
 				pkField := reflect.Indirect(fieldValue).FieldByName(fieldTable.PKColumns()[0].FieldName)
 				return pkField.Interface(), nil
@@ -614,14 +606,14 @@ func (session *Session) value2Interface(col *core.Column, fieldValue reflect.Val
 		if col.SQLType.IsText() {
 			bytes, err := json.Marshal(fieldValue.Interface())
 			if err != nil {
-				session.Engine.logger.Error(err)
+				session.engine.logger.Error(err)
 				return 0, err
 			}
 			return string(bytes), nil
 		} else if col.SQLType.IsBlob() {
 			bytes, err := json.Marshal(fieldValue.Interface())
 			if err != nil {
-				session.Engine.logger.Error(err)
+				session.engine.logger.Error(err)
 				return 0, err
 			}
 			return bytes, nil
@@ -630,7 +622,7 @@ func (session *Session) value2Interface(col *core.Column, fieldValue reflect.Val
 	case reflect.Complex64, reflect.Complex128:
 		bytes, err := json.Marshal(fieldValue.Interface())
 		if err != nil {
-			session.Engine.logger.Error(err)
+			session.engine.logger.Error(err)
 			return 0, err
 		}
 		return string(bytes), nil
@@ -642,7 +634,7 @@ func (session *Session) value2Interface(col *core.Column, fieldValue reflect.Val
 		if col.SQLType.IsText() {
 			bytes, err := json.Marshal(fieldValue.Interface())
 			if err != nil {
-				session.Engine.logger.Error(err)
+				session.engine.logger.Error(err)
 				return 0, err
 			}
 			return string(bytes), nil
@@ -655,7 +647,7 @@ func (session *Session) value2Interface(col *core.Column, fieldValue reflect.Val
 			} else {
 				bytes, err = json.Marshal(fieldValue.Interface())
 				if err != nil {
-					session.Engine.logger.Error(err)
+					session.engine.logger.Error(err)
 					return 0, err
 				}
 			}

vendor/github.com/go-xorm/xorm/session_delete.go

@@ -12,26 +12,26 @@ import (
 	"github.com/go-xorm/core"
 )
 
-func (session *Session) cacheDelete(sqlStr string, args ...interface{}) error {
+func (session *Session) cacheDelete(table *core.Table, tableName, sqlStr string, args ...interface{}) error {
-	if session.Statement.RefTable == nil ||
+	if table == nil ||
-		session.Tx != nil {
+		session.tx != nil {
 		return ErrCacheFailed
 	}
 
-	for _, filter := range session.Engine.dialect.Filters() {
+	for _, filter := range session.engine.dialect.Filters() {
-		sqlStr = filter.Do(sqlStr, session.Engine.dialect, session.Statement.RefTable)
+		sqlStr = filter.Do(sqlStr, session.engine.dialect, table)
 	}
 
-	newsql := session.Statement.convertIDSQL(sqlStr)
+	newsql := session.statement.convertIDSQL(sqlStr)
 	if newsql == "" {
 		return ErrCacheFailed
 	}
 
-	cacher := session.Engine.getCacher2(session.Statement.RefTable)
+	cacher := session.engine.getCacher2(table)
-	tableName := session.Statement.TableName()
+	pkColumns := table.PKColumns()
 	ids, err := core.GetCacheSql(cacher, tableName, newsql, args)
 	if err != nil {
-		resultsSlice, err := session.query(newsql, args...)
+		resultsSlice, err := session.queryBytes(newsql, args...)
 		if err != nil {
 			return err
 		}
@@ -40,7 +40,7 @@ func (session *Session) cacheDelete(sqlStr string, args ...interface{}) error {
 			for _, data := range resultsSlice {
 				var id int64
 				var pk core.PK = make([]interface{}, 0)
-				for _, col := range session.Statement.RefTable.PKColumns() {
+				for _, col := range pkColumns {
 					if v, ok := data[col.Name]; !ok {
 						return errors.New("no id")
 					} else if col.SQLType.IsText() {
@@ -58,33 +58,30 @@ func (session *Session) cacheDelete(sqlStr string, args ...interface{}) error {
 				ids = append(ids, pk)
 			}
 		}
-	} /*else {
+	}
-	    session.Engine.LogDebug("delete cache sql %v", newsql)
-	    cacher.DelIds(tableName, genSqlKey(newsql, args))
-	}*/
 
 	for _, id := range ids {
-		session.Engine.logger.Debug("[cacheDelete] delete cache obj", tableName, id)
+		session.engine.logger.Debug("[cacheDelete] delete cache obj:", tableName, id)
 		sid, err := id.ToString()
 		if err != nil {
 			return err
 		}
 		cacher.DelBean(tableName, sid)
 	}
-	session.Engine.logger.Debug("[cacheDelete] clear cache sql", tableName)
+	session.engine.logger.Debug("[cacheDelete] clear cache table:", tableName)
 	cacher.ClearIds(tableName)
 	return nil
 }
 
 // Delete records, bean's non-empty fields are conditions
 func (session *Session) Delete(bean interface{}) (int64, error) {
-	defer session.resetStatement()
+	if session.isAutoClose {
-	if session.IsAutoClose {
 		defer session.Close()
 	}
 
-	session.Statement.setRefValue(rValue(bean))
+	if err := session.statement.setRefValue(rValue(bean)); err != nil {
-	var table = session.Statement.RefTable
+		return 0, err
+	}
 
 	// handle before delete processors
 	for _, closure := range session.beforeClosures {
@@ -96,13 +93,17 @@ func (session *Session) Delete(bean interface{}) (int64, error) {
 		processor.BeforeDelete()
 	}
 
-	// --
+	condSQL, condArgs, err := session.statement.genConds(bean)
-	condSQL, condArgs, _ := session.Statement.genConds(bean)
+	if err != nil {
-	if len(condSQL) == 0 && session.Statement.LimitN == 0 {
+		return 0, err
+	}
+	if len(condSQL) == 0 && session.statement.LimitN == 0 {
 		return 0, ErrNeedDeletedCond
 	}
 
-	var tableName = session.Engine.Quote(session.Statement.TableName())
+	var tableNameNoQuote = session.statement.TableName()
+	var tableName = session.engine.Quote(tableNameNoQuote)
+	var table = session.statement.RefTable
 	var deleteSQL string
 	if len(condSQL) > 0 {
 		deleteSQL = fmt.Sprintf("DELETE FROM %v WHERE %v", tableName, condSQL)
@@ -111,15 +112,15 @@ func (session *Session) Delete(bean interface{}) (int64, error) {
 	}
 
 	var orderSQL string
-	if len(session.Statement.OrderStr) > 0 {
+	if len(session.statement.OrderStr) > 0 {
-		orderSQL += fmt.Sprintf(" ORDER BY %s", session.Statement.OrderStr)
+		orderSQL += fmt.Sprintf(" ORDER BY %s", session.statement.OrderStr)
 	}
-	if session.Statement.LimitN > 0 {
+	if session.statement.LimitN > 0 {
-		orderSQL += fmt.Sprintf(" LIMIT %d", session.Statement.LimitN)
+		orderSQL += fmt.Sprintf(" LIMIT %d", session.statement.LimitN)
 	}
 
 	if len(orderSQL) > 0 {
-		switch session.Engine.dialect.DBType() {
+		switch session.engine.dialect.DBType() {
 		case core.POSTGRES:
 			inSQL := fmt.Sprintf("ctid IN (SELECT ctid FROM %s%s)", tableName, orderSQL)
 			if len(condSQL) > 0 {
@@ -144,7 +145,7 @@ func (session *Session) Delete(bean interface{}) (int64, error) {
 
 	var realSQL string
 	argsForCache := make([]interface{}, 0, len(condArgs)*2)
-	if session.Statement.unscoped || table.DeletedColumn() == nil { // tag "deleted" is disabled
+	if session.statement.unscoped || table.DeletedColumn() == nil { // tag "deleted" is disabled
 		realSQL = deleteSQL
 		copy(argsForCache, condArgs)
 		argsForCache = append(condArgs, argsForCache...)
@@ -155,12 +156,12 @@ func (session *Session) Delete(bean interface{}) (int64, error) {
 
 		deletedColumn := table.DeletedColumn()
 		realSQL = fmt.Sprintf("UPDATE %v SET %v = ? WHERE %v",
-			session.Engine.Quote(session.Statement.TableName()),
+			session.engine.Quote(session.statement.TableName()),
-			session.Engine.Quote(deletedColumn.Name),
+			session.engine.Quote(deletedColumn.Name),
 			condSQL)
 
 		if len(orderSQL) > 0 {
-			switch session.Engine.dialect.DBType() {
+			switch session.engine.dialect.DBType() {
 			case core.POSTGRES:
 				inSQL := fmt.Sprintf("ctid IN (SELECT ctid FROM %s%s)", tableName, orderSQL)
 				if len(condSQL) > 0 {
@@ -183,12 +184,12 @@ func (session *Session) Delete(bean interface{}) (int64, error) {
 			}
 		}
 
-		// !oinume! Insert NowTime to the head of session.Statement.Params
+		// !oinume! Insert nowTime to the head of session.statement.Params
 		condArgs = append(condArgs, "")
 		paramsLen := len(condArgs)
 		copy(condArgs[1:paramsLen], condArgs[0:paramsLen-1])
 
-		val, t := session.Engine.NowTime2(deletedColumn.SQLType.Name)
+		val, t := session.engine.nowTime(deletedColumn)
 		condArgs[0] = val
 
 		var colName = deletedColumn.Name
@@ -198,17 +199,18 @@ func (session *Session) Delete(bean interface{}) (int64, error) {
 		})
 	}
 
-	if cacher := session.Engine.getCacher2(session.Statement.RefTable); cacher != nil && session.Statement.UseCache {
+	if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache {
-		session.cacheDelete(deleteSQL, argsForCache...)
+		session.cacheDelete(table, tableNameNoQuote, deleteSQL, argsForCache...)
 	}
 
+	session.statement.RefTable = table
 	res, err := session.exec(realSQL, condArgs...)
 	if err != nil {
 		return 0, err
 	}
 
 	// handle after delete processors
-	if session.IsAutoCommit {
+	if session.isAutoCommit {
 		for _, closure := range session.afterClosures {
 			closure(bean)
 		}

vendor/github.com/go-xorm/xorm/session_exist.go

@@ -0,0 +1,77 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package xorm
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+
+	"github.com/go-xorm/builder"
+)
+
+// Exist returns true if the record exist otherwise return false
+func (session *Session) Exist(bean ...interface{}) (bool, error) {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	var sqlStr string
+	var args []interface{}
+	var err error
+
+	if session.statement.RawSQL == "" {
+		if len(bean) == 0 {
+			tableName := session.statement.TableName()
+			if len(tableName) <= 0 {
+				return false, ErrTableNotFound
+			}
+
+			if session.statement.cond.IsValid() {
+				condSQL, condArgs, err := builder.ToSQL(session.statement.cond)
+				if err != nil {
+					return false, err
+				}
+
+				sqlStr = fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT 1", tableName, condSQL)
+				args = condArgs
+			} else {
+				sqlStr = fmt.Sprintf("SELECT * FROM %s LIMIT 1", tableName)
+				args = []interface{}{}
+			}
+		} else {
+			beanValue := reflect.ValueOf(bean[0])
+			if beanValue.Kind() != reflect.Ptr {
+				return false, errors.New("needs a pointer")
+			}
+
+			if beanValue.Elem().Kind() == reflect.Struct {
+				if err := session.statement.setRefValue(beanValue.Elem()); err != nil {
+					return false, err
+				}
+			}
+
+			if len(session.statement.TableName()) <= 0 {
+				return false, ErrTableNotFound
+			}
+			session.statement.Limit(1)
+			sqlStr, args, err = session.statement.genGetSQL(bean[0])
+			if err != nil {
+				return false, err
+			}
+		}
+	} else {
+		sqlStr = session.statement.RawSQL
+		args = session.statement.RawParams
+	}
+
+	rows, err := session.queryRows(sqlStr, args...)
+	if err != nil {
+		return false, err
+	}
+	defer rows.Close()
+
+	return rows.Next(), nil
+}

vendor/github.com/go-xorm/xorm/session_find.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
-	"strconv"
 	"strings"
 
 	"github.com/go-xorm/builder"
@@ -24,11 +23,13 @@ const (
 // are conditions. beans could be []Struct, []*Struct, map[int64]Struct
 // map[int64]*Struct
 func (session *Session) Find(rowsSlicePtr interface{}, condiBean ...interface{}) error {
-	defer session.resetStatement()
+	if session.isAutoClose {
-	if session.IsAutoClose {
 		defer session.Close()
 	}
+	return session.find(rowsSlicePtr, condiBean...)
+}
 
+func (session *Session) find(rowsSlicePtr interface{}, condiBean ...interface{}) error {
 	sliceValue := reflect.Indirect(reflect.ValueOf(rowsSlicePtr))
 	if sliceValue.Kind() != reflect.Slice && sliceValue.Kind() != reflect.Map {
 		return errors.New("needs a pointer to a slice or a map")
@@ -37,77 +38,79 @@ func (session *Session) Find(rowsSlicePtr interface{}, condiBean ...interface{})
 	sliceElementType := sliceValue.Type().Elem()
 
 	var tp = tpStruct
-	if session.Statement.RefTable == nil {
+	if session.statement.RefTable == nil {
 		if sliceElementType.Kind() == reflect.Ptr {
 			if sliceElementType.Elem().Kind() == reflect.Struct {
 				pv := reflect.New(sliceElementType.Elem())
-				session.Statement.setRefValue(pv.Elem())
+				if err := session.statement.setRefValue(pv.Elem()); err != nil {
+					return err
+				}
 			} else {
 				tp = tpNonStruct
 			}
 		} else if sliceElementType.Kind() == reflect.Struct {
 			pv := reflect.New(sliceElementType)
-			session.Statement.setRefValue(pv.Elem())
+			if err := session.statement.setRefValue(pv.Elem()); err != nil {
+				return err
+			}
 		} else {
 			tp = tpNonStruct
 		}
 	}
 
-	var table = session.Statement.RefTable
+	var table = session.statement.RefTable
 
-	var addedTableName = (len(session.Statement.JoinStr) > 0)
+	var addedTableName = (len(session.statement.JoinStr) > 0)
 	var autoCond builder.Cond
 	if tp == tpStruct {
-		if !session.Statement.noAutoCondition && len(condiBean) > 0 {
+		if !session.statement.noAutoCondition && len(condiBean) > 0 {
 			var err error
-			autoCond, err = session.Statement.buildConds(table, condiBean[0], true, true, false, true, addedTableName)
+			autoCond, err = session.statement.buildConds(table, condiBean[0], true, true, false, true, addedTableName)
 			if err != nil {
-				panic(err)
+				return err
 			}
 		} else {
 			// !oinume! Add "<col> IS NULL" to WHERE whatever condiBean is given.
 			// See https://github.com/go-xorm/xorm/issues/179
-			if col := table.DeletedColumn(); col != nil && !session.Statement.unscoped { // tag "deleted" is enabled
+			if col := table.DeletedColumn(); col != nil && !session.statement.unscoped { // tag "deleted" is enabled
-				var colName = session.Engine.Quote(col.Name)
+				var colName = session.engine.Quote(col.Name)
 				if addedTableName {
-					var nm = session.Statement.TableName()
+					var nm = session.statement.TableName()
-					if len(session.Statement.TableAlias) > 0 {
+					if len(session.statement.TableAlias) > 0 {
-						nm = session.Statement.TableAlias
+						nm = session.statement.TableAlias
 					}
-					colName = session.Engine.Quote(nm) + "." + colName
+					colName = session.engine.Quote(nm) + "." + colName
-				}
-				if session.Engine.dialect.DBType() == core.MSSQL {
-					autoCond = builder.IsNull{colName}
-				} else {
-					autoCond = builder.IsNull{colName}.Or(builder.Eq{colName: "0001-01-01 00:00:00"})
 				}
+
+				autoCond = session.engine.CondDeleted(colName)
 			}
 		}
 	}
 
 	var sqlStr string
 	var args []interface{}
-	if session.Statement.RawSQL == "" {
+	var err error
-		if len(session.Statement.TableName()) <= 0 {
+	if session.statement.RawSQL == "" {
+		if len(session.statement.TableName()) <= 0 {
 			return ErrTableNotFound
 		}
 
-		var columnStr = session.Statement.ColumnStr
+		var columnStr = session.statement.ColumnStr
-		if len(session.Statement.selectStr) > 0 {
+		if len(session.statement.selectStr) > 0 {
-			columnStr = session.Statement.selectStr
+			columnStr = session.statement.selectStr
 		} else {
-			if session.Statement.JoinStr == "" {
+			if session.statement.JoinStr == "" {
 				if columnStr == "" {
-					if session.Statement.GroupByStr != "" {
+					if session.statement.GroupByStr != "" {
-						columnStr = session.Statement.Engine.Quote(strings.Replace(session.Statement.GroupByStr, ",", session.Engine.Quote(","), -1))
+						columnStr = session.statement.Engine.Quote(strings.Replace(session.statement.GroupByStr, ",", session.engine.Quote(","), -1))
 					} else {
-						columnStr = session.Statement.genColumnStr()
+						columnStr = session.statement.genColumnStr()
 					}
 				}
 			} else {
 				if columnStr == "" {
-					if session.Statement.GroupByStr != "" {
+					if session.statement.GroupByStr != "" {
-						columnStr = session.Statement.Engine.Quote(strings.Replace(session.Statement.GroupByStr, ",", session.Engine.Quote(","), -1))
+						columnStr = session.statement.Engine.Quote(strings.Replace(session.statement.GroupByStr, ",", session.engine.Quote(","), -1))
 					} else {
 						columnStr = "*"
 					}
@@ -118,31 +121,37 @@ func (session *Session) Find(rowsSlicePtr interface{}, condiBean ...interface{})
 			}
 		}
 
-		condSQL, condArgs, _ := builder.ToSQL(session.Statement.cond.And(autoCond))
+		session.statement.cond = session.statement.cond.And(autoCond)
+		condSQL, condArgs, err := builder.ToSQL(session.statement.cond)
+		if err != nil {
+			return err
+		}
 
-		args = append(session.Statement.joinArgs, condArgs...)
+		args = append(session.statement.joinArgs, condArgs...)
-		sqlStr = session.Statement.genSelectSQL(columnStr, condSQL)
+		sqlStr, err = session.statement.genSelectSQL(columnStr, condSQL)
+		if err != nil {
+			return err
+		}
 		// for mssql and use limit
 		qs := strings.Count(sqlStr, "?")
 		if len(args)*2 == qs {
 			args = append(args, args...)
 		}
 	} else {
-		sqlStr = session.Statement.RawSQL
+		sqlStr = session.statement.RawSQL
-		args = session.Statement.RawParams
+		args = session.statement.RawParams
 	}
 
-	var err error
 	if session.canCache() {
-		if cacher := session.Engine.getCacher2(table); cacher != nil &&
+		if cacher := session.engine.getCacher2(table); cacher != nil &&
-			!session.Statement.IsDistinct &&
+			!session.statement.IsDistinct &&
-			!session.Statement.unscoped {
+			!session.statement.unscoped {
 			err = session.cacheFind(sliceElementType, sqlStr, rowsSlicePtr, args...)
 			if err != ErrCacheFailed {
 				return err
 			}
 			err = nil // !nashtsai! reset err to nil for ErrCacheFailed
-			session.Engine.logger.Warn("Cache Find Failed")
+			session.engine.logger.Warn("Cache Find Failed")
 		}
 	}
 
@@ -150,21 +159,13 @@ func (session *Session) Find(rowsSlicePtr interface{}, condiBean ...interface{})
 }
 
 func (session *Session) noCacheFind(table *core.Table, containerValue reflect.Value, sqlStr string, args ...interface{}) error {
-	var rawRows *core.Rows
+	rows, err := session.queryRows(sqlStr, args...)
-	var err error
-
-	session.queryPreprocess(&sqlStr, args...)
-	if session.IsAutoCommit {
-		_, rawRows, err = session.innerQuery(sqlStr, args...)
-	} else {
-		rawRows, err = session.Tx.Query(sqlStr, args...)
-	}
 	if err != nil {
 		return err
 	}
-	defer rawRows.Close()
+	defer rows.Close()
 
-	fields, err := rawRows.Columns()
+	fields, err := rows.Columns()
 	if err != nil {
 		return err
 	}
@@ -234,20 +235,29 @@ func (session *Session) noCacheFind(table *core.Table, containerValue reflect.Va
 	if elemType.Kind() == reflect.Struct {
 		var newValue = newElemFunc(fields)
 		dataStruct := rValue(newValue.Interface())
-		return session.rows2Beans(rawRows, fields, len(fields), session.Engine.autoMapType(dataStruct), newElemFunc, containerValueSetFunc)
+		tb, err := session.engine.autoMapType(dataStruct)
+		if err != nil {
+			return err
+		}
+		err = session.rows2Beans(rows, fields, tb, newElemFunc, containerValueSetFunc)
+		rows.Close()
+		if err != nil {
+			return err
+		}
+		return session.executeProcessors()
 	}
 
-	for rawRows.Next() {
+	for rows.Next() {
 		var newValue = newElemFunc(fields)
 		bean := newValue.Interface()
 
 		switch elemType.Kind() {
 		case reflect.Slice:
-			err = rawRows.ScanSlice(bean)
+			err = rows.ScanSlice(bean)
 		case reflect.Map:
-			err = rawRows.ScanMap(bean)
+			err = rows.ScanMap(bean)
 		default:
-			err = rawRows.Scan(bean)
+			err = rows.Scan(bean)
 		}
 
 		if err != nil {
@@ -278,22 +288,21 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 		return ErrCacheFailed
 	}
 
-	for _, filter := range session.Engine.dialect.Filters() {
+	for _, filter := range session.engine.dialect.Filters() {
-		sqlStr = filter.Do(sqlStr, session.Engine.dialect, session.Statement.RefTable)
+		sqlStr = filter.Do(sqlStr, session.engine.dialect, session.statement.RefTable)
 	}
 
-	newsql := session.Statement.convertIDSQL(sqlStr)
+	newsql := session.statement.convertIDSQL(sqlStr)
 	if newsql == "" {
 		return ErrCacheFailed
 	}
 
-	tableName := session.Statement.TableName()
+	tableName := session.statement.TableName()
-
+	table := session.statement.RefTable
-	table := session.Statement.RefTable
+	cacher := session.engine.getCacher2(table)
-	cacher := session.Engine.getCacher2(table)
 	ids, err := core.GetCacheSql(cacher, tableName, newsql, args)
 	if err != nil {
-		rows, err := session.DB().Query(newsql, args...)
+		rows, err := session.queryRows(newsql, args...)
 		if err != nil {
 			return err
 		}
@@ -304,7 +313,7 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 		for rows.Next() {
 			i++
 			if i > 500 {
-				session.Engine.logger.Debug("[cacheFind] ids length > 500, no cache")
+				session.engine.logger.Debug("[cacheFind] ids length > 500, no cache")
 				return ErrCacheFailed
 			}
 			var res = make([]string, len(table.PrimaryKeys))
@@ -312,32 +321,24 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 			if err != nil {
 				return err
 			}
-
 			var pk core.PK = make([]interface{}, len(table.PrimaryKeys))
 			for i, col := range table.PKColumns() {
-				if col.SQLType.IsNumeric() {
+				pk[i], err = session.engine.idTypeAssertion(col, res[i])
-					n, err := strconv.ParseInt(res[i], 10, 64)
+				if err != nil {
-					if err != nil {
+					return err
-						return err
-					}
-					pk[i] = n
-				} else if col.SQLType.IsText() {
-					pk[i] = res[i]
-				} else {
-					return errors.New("not supported")
 				}
 			}
 
 			ids = append(ids, pk)
 		}
 
-		session.Engine.logger.Debug("[cacheFind] cache sql:", ids, tableName, newsql, args)
+		session.engine.logger.Debug("[cacheFind] cache sql:", ids, tableName, sqlStr, newsql, args)
 		err = core.PutCacheSql(cacher, ids, tableName, newsql, args)
 		if err != nil {
 			return err
 		}
 	} else {
-		session.Engine.logger.Debug("[cacheFind] cache hit sql:", newsql, args)
+		session.engine.logger.Debug("[cacheFind] cache hit sql:", tableName, sqlStr, newsql, args)
 	}
 
 	sliceValue := reflect.Indirect(reflect.ValueOf(rowsSlicePtr))
@@ -352,20 +353,20 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 			return err
 		}
 		bean := cacher.GetBean(tableName, sid)
-		if bean == nil {
+		if bean == nil || reflect.ValueOf(bean).Elem().Type() != t {
 			ides = append(ides, id)
 			ididxes[sid] = idx
 		} else {
-			session.Engine.logger.Debug("[cacheFind] cache hit bean:", tableName, id, bean)
+			session.engine.logger.Debug("[cacheFind] cache hit bean:", tableName, id, bean)
 
-			pk := session.Engine.IdOf(bean)
+			pk := session.engine.IdOf(bean)
 			xid, err := pk.ToString()
 			if err != nil {
 				return err
 			}
 
 			if sid != xid {
-				session.Engine.logger.Error("[cacheFind] error cache", xid, sid, bean)
+				session.engine.logger.Error("[cacheFind] error cache", xid, sid, bean)
 				return ErrCacheFailed
 			}
 			temps[idx] = bean
@@ -373,9 +374,6 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 	}
 
 	if len(ides) > 0 {
-		newSession := session.Engine.NewSession()
-		defer newSession.Close()
-
 		slices := reflect.New(reflect.SliceOf(t))
 		beans := slices.Interface()
 
@@ -385,18 +383,18 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 				ff = append(ff, ie[0])
 			}
 
-			newSession.In("`"+table.PrimaryKeys[0]+"`", ff...)
+			session.In("`"+table.PrimaryKeys[0]+"`", ff...)
 		} else {
 			for _, ie := range ides {
 				cond := builder.NewCond()
 				for i, name := range table.PrimaryKeys {
 					cond = cond.And(builder.Eq{"`" + name + "`": ie[i]})
 				}
-				newSession.Or(cond)
+				session.Or(cond)
 			}
 		}
 
-		err = newSession.NoCache().Find(beans)
+		err = session.NoCache().Table(tableName).find(beans)
 		if err != nil {
 			return err
 		}
@@ -407,7 +405,10 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 			if rv.Kind() != reflect.Ptr {
 				rv = rv.Addr()
 			}
-			id := session.Engine.IdOfV(rv)
+			id, err := session.engine.idOfV(rv)
+			if err != nil {
+				return err
+			}
 			sid, err := id.ToString()
 			if err != nil {
 				return err
@@ -415,7 +416,7 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 
 			bean := rv.Interface()
 			temps[ididxes[sid]] = bean
-			session.Engine.logger.Debug("[cacheFind] cache bean:", tableName, id, bean, temps)
+			session.engine.logger.Debug("[cacheFind] cache bean:", tableName, id, bean, temps)
 			cacher.PutBean(tableName, sid, bean)
 		}
 	}
@@ -423,7 +424,7 @@ func (session *Session) cacheFind(t reflect.Type, sqlStr string, rowsSlicePtr in
 	for j := 0; j < len(temps); j++ {
 		bean := temps[j]
 		if bean == nil {
-			session.Engine.logger.Warn("[cacheFind] cache no hit:", tableName, ids[j], temps)
+			session.engine.logger.Warn("[cacheFind] cache no hit:", tableName, ids[j], temps)
 			// return errors.New("cache error") // !nashtsai! no need to return error, but continue instead
 			continue
 		}

vendor/github.com/go-xorm/xorm/session_get.go

@@ -15,42 +15,49 @@ import (
 // Get retrieve one record from database, bean's non-empty fields
 // will be as conditions
 func (session *Session) Get(bean interface{}) (bool, error) {
-	defer session.resetStatement()
+	if session.isAutoClose {
-	if session.IsAutoClose {
 		defer session.Close()
 	}
+	return session.get(bean)
+}
 
+func (session *Session) get(bean interface{}) (bool, error) {
 	beanValue := reflect.ValueOf(bean)
 	if beanValue.Kind() != reflect.Ptr {
-		return false, errors.New("needs a pointer to a struct")
+		return false, errors.New("needs a pointer to a value")
-	}
+	} else if beanValue.Elem().Kind() == reflect.Ptr {
-
+		return false, errors.New("a pointer to a pointer is not allowed")
-	// FIXME: remove this after support non-struct Get
-	if beanValue.Elem().Kind() != reflect.Struct {
-		return false, errors.New("needs a pointer to a struct")
 	}
 
 	if beanValue.Elem().Kind() == reflect.Struct {
-		session.Statement.setRefValue(beanValue.Elem())
+		if err := session.statement.setRefValue(beanValue.Elem()); err != nil {
+			return false, err
+		}
 	}
 
 	var sqlStr string
 	var args []interface{}
+	var err error
 
-	if session.Statement.RawSQL == "" {
+	if session.statement.RawSQL == "" {
-		if len(session.Statement.TableName()) <= 0 {
+		if len(session.statement.TableName()) <= 0 {
 			return false, ErrTableNotFound
 		}
-		session.Statement.Limit(1)
+		session.statement.Limit(1)
-		sqlStr, args = session.Statement.genGetSQL(bean)
+		sqlStr, args, err = session.statement.genGetSQL(bean)
+		if err != nil {
+			return false, err
+		}
 	} else {
-		sqlStr = session.Statement.RawSQL
+		sqlStr = session.statement.RawSQL
-		args = session.Statement.RawParams
+		args = session.statement.RawParams
 	}
 
-	if session.canCache() {
+	table := session.statement.RefTable
-		if cacher := session.Engine.getCacher2(session.Statement.RefTable); cacher != nil &&
+
-			!session.Statement.unscoped {
+	if session.canCache() && beanValue.Elem().Kind() == reflect.Struct {
+		if cacher := session.engine.getCacher2(table); cacher != nil &&
+			!session.statement.unscoped {
 			has, err := session.cacheGet(bean, sqlStr, args...)
 			if err != ErrCacheFailed {
 				return has, err
@@ -58,47 +65,51 @@ func (session *Session) Get(bean interface{}) (bool, error) {
 		}
 	}
 
-	return session.nocacheGet(beanValue.Elem().Kind(), bean, sqlStr, args...)
+	return session.nocacheGet(beanValue.Elem().Kind(), table, bean, sqlStr, args...)
 }
 
-func (session *Session) nocacheGet(beanKind reflect.Kind, bean interface{}, sqlStr string, args ...interface{}) (bool, error) {
+func (session *Session) nocacheGet(beanKind reflect.Kind, table *core.Table, bean interface{}, sqlStr string, args ...interface{}) (bool, error) {
-	var rawRows *core.Rows
+	rows, err := session.queryRows(sqlStr, args...)
-	var err error
-	session.queryPreprocess(&sqlStr, args...)
-	if session.IsAutoCommit {
-		_, rawRows, err = session.innerQuery(sqlStr, args...)
-	} else {
-		rawRows, err = session.Tx.Query(sqlStr, args...)
-	}
 	if err != nil {
 		return false, err
 	}
+	defer rows.Close()
 
-	defer rawRows.Close()
+	if !rows.Next() {
+		return false, nil
+	}
 
-	if rawRows.Next() {
+	switch beanKind {
-		fields, err := rawRows.Columns()
+	case reflect.Struct:
+		fields, err := rows.Columns()
 		if err != nil {
-			// WARN: Alougth rawRows return true, but get fields failed
+			// WARN: Alougth rows return true, but get fields failed
 			return true, err
 		}
 
-		switch beanKind {
+		scanResults, err := session.row2Slice(rows, fields, bean)
-		case reflect.Struct:
+		if err != nil {
-			dataStruct := rValue(bean)
+			return false, err
-			session.Statement.setRefValue(dataStruct)
+		}
-			_, err = session.row2Bean(rawRows, fields, len(fields), bean, &dataStruct, session.Statement.RefTable)
+		// close it before covert data
-		case reflect.Slice:
+		rows.Close()
-			err = rawRows.ScanSlice(bean)
+
-		case reflect.Map:
+		dataStruct := rValue(bean)
-			err = rawRows.ScanMap(bean)
+		_, err = session.slice2Bean(scanResults, fields, bean, &dataStruct, table)
-		default:
+		if err != nil {
-			err = rawRows.Scan(bean)
+			return true, err
 		}
 
-		return true, err
+		return true, session.executeProcessors()
+	case reflect.Slice:
+		err = rows.ScanSlice(bean)
+	case reflect.Map:
+		err = rows.ScanMap(bean)
+	default:
+		err = rows.Scan(bean)
 	}
-	return false, nil
+
+	return true, err
 }
 
 func (session *Session) cacheGet(bean interface{}, sqlStr string, args ...interface{}) (has bool, err error) {
@@ -107,22 +118,22 @@ func (session *Session) cacheGet(bean interface{}, sqlStr string, args ...interf
 		return false, ErrCacheFailed
 	}
 
-	for _, filter := range session.Engine.dialect.Filters() {
+	for _, filter := range session.engine.dialect.Filters() {
-		sqlStr = filter.Do(sqlStr, session.Engine.dialect, session.Statement.RefTable)
+		sqlStr = filter.Do(sqlStr, session.engine.dialect, session.statement.RefTable)
 	}
-	newsql := session.Statement.convertIDSQL(sqlStr)
+	newsql := session.statement.convertIDSQL(sqlStr)
 	if newsql == "" {
 		return false, ErrCacheFailed
 	}
 
-	cacher := session.Engine.getCacher2(session.Statement.RefTable)
+	cacher := session.engine.getCacher2(session.statement.RefTable)
-	tableName := session.Statement.TableName()
+	tableName := session.statement.TableName()
-	session.Engine.logger.Debug("[cacheGet] find sql:", newsql, args)
+	session.engine.logger.Debug("[cacheGet] find sql:", newsql, args)
+	table := session.statement.RefTable
 	ids, err := core.GetCacheSql(cacher, tableName, newsql, args)
-	table := session.Statement.RefTable
 	if err != nil {
 		var res = make([]string, len(table.PrimaryKeys))
-		rows, err := session.DB().Query(newsql, args...)
+		rows, err := session.NoCache().queryRows(newsql, args...)
 		if err != nil {
 			return false, err
 		}
@@ -153,19 +164,19 @@ func (session *Session) cacheGet(bean interface{}, sqlStr string, args ...interf
 		}
 
 		ids = []core.PK{pk}
-		session.Engine.logger.Debug("[cacheGet] cache ids:", newsql, ids)
+		session.engine.logger.Debug("[cacheGet] cache ids:", newsql, ids)
 		err = core.PutCacheSql(cacher, ids, tableName, newsql, args)
 		if err != nil {
 			return false, err
 		}
 	} else {
-		session.Engine.logger.Debug("[cacheGet] cache hit sql:", newsql)
+		session.engine.logger.Debug("[cacheGet] cache hit sql:", newsql, ids)
 	}
 
 	if len(ids) > 0 {
 		structValue := reflect.Indirect(reflect.ValueOf(bean))
 		id := ids[0]
-		session.Engine.logger.Debug("[cacheGet] get bean:", tableName, id)
+		session.engine.logger.Debug("[cacheGet] get bean:", tableName, id)
 		sid, err := id.ToString()
 		if err != nil {
 			return false, err
@@ -173,15 +184,15 @@ func (session *Session) cacheGet(bean interface{}, sqlStr string, args ...interf
 		cacheBean := cacher.GetBean(tableName, sid)
 		if cacheBean == nil {
 			cacheBean = bean
-			has, err = session.nocacheGet(reflect.Struct, cacheBean, sqlStr, args...)
+			has, err = session.nocacheGet(reflect.Struct, table, cacheBean, sqlStr, args...)
 			if err != nil || !has {
 				return has, err
 			}
 
-			session.Engine.logger.Debug("[cacheGet] cache bean:", tableName, id, cacheBean)
+			session.engine.logger.Debug("[cacheGet] cache bean:", tableName, id, cacheBean)
 			cacher.PutBean(tableName, sid, cacheBean)
 		} else {
-			session.Engine.logger.Debug("[cacheGet] cache hit bean:", tableName, id, cacheBean)
+			session.engine.logger.Debug("[cacheGet] cache hit bean:", tableName, id, cacheBean)
 			has = true
 		}
 		structValue.Set(reflect.Indirect(reflect.ValueOf(cacheBean)))

vendor/github.com/go-xorm/xorm/session_insert.go

@@ -19,17 +19,16 @@ func (session *Session) Insert(beans ...interface{}) (int64, error) {
 	var affected int64
 	var err error
 
-	if session.IsAutoClose {
+	if session.isAutoClose {
 		defer session.Close()
 	}
-	defer session.resetStatement()
 
 	for _, bean := range beans {
 		sliceValue := reflect.Indirect(reflect.ValueOf(bean))
 		if sliceValue.Kind() == reflect.Slice {
 			size := sliceValue.Len()
 			if size > 0 {
-				if session.Engine.SupportInsertMany() {
+				if session.engine.SupportInsertMany() {
 					cnt, err := session.innerInsertMulti(bean)
 					if err != nil {
 						return affected, err
@@ -67,13 +66,15 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 		return 0, errors.New("could not insert a empty slice")
 	}
 
-	session.Statement.setRefValue(sliceValue.Index(0))
+	if err := session.statement.setRefValue(reflect.ValueOf(sliceValue.Index(0).Interface())); err != nil {
+		return 0, err
+	}
 
-	if len(session.Statement.TableName()) <= 0 {
+	if len(session.statement.TableName()) <= 0 {
 		return 0, ErrTableNotFound
 	}
 
-	table := session.Statement.RefTable
+	table := session.statement.RefTable
 	size := sliceValue.Len()
 
 	var colNames []string
@@ -114,18 +115,18 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 				if col.IsDeleted {
 					continue
 				}
-				if session.Statement.ColumnStr != "" {
+				if session.statement.ColumnStr != "" {
-					if _, ok := getFlagForColumn(session.Statement.columnMap, col); !ok {
+					if _, ok := getFlagForColumn(session.statement.columnMap, col); !ok {
 						continue
 					}
 				}
-				if session.Statement.OmitStr != "" {
+				if session.statement.OmitStr != "" {
-					if _, ok := getFlagForColumn(session.Statement.columnMap, col); ok {
+					if _, ok := getFlagForColumn(session.statement.columnMap, col); ok {
 						continue
 					}
 				}
-				if (col.IsCreated || col.IsUpdated) && session.Statement.UseAutoTime {
+				if (col.IsCreated || col.IsUpdated) && session.statement.UseAutoTime {
-					val, t := session.Engine.NowTime2(col.SQLType.Name)
+					val, t := session.engine.nowTime(col)
 					args = append(args, val)
 
 					var colName = col.Name
@@ -133,7 +134,7 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 						col := table.GetColumn(colName)
 						setColumnTime(bean, col, t)
 					})
-				} else if col.IsVersion && session.Statement.checkVersion {
+				} else if col.IsVersion && session.statement.checkVersion {
 					args = append(args, 1)
 					var colName = col.Name
 					session.afterClosures = append(session.afterClosures, func(bean interface{}) {
@@ -169,18 +170,18 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 				if col.IsDeleted {
 					continue
 				}
-				if session.Statement.ColumnStr != "" {
+				if session.statement.ColumnStr != "" {
-					if _, ok := getFlagForColumn(session.Statement.columnMap, col); !ok {
+					if _, ok := getFlagForColumn(session.statement.columnMap, col); !ok {
 						continue
 					}
 				}
-				if session.Statement.OmitStr != "" {
+				if session.statement.OmitStr != "" {
-					if _, ok := getFlagForColumn(session.Statement.columnMap, col); ok {
+					if _, ok := getFlagForColumn(session.statement.columnMap, col); ok {
 						continue
 					}
 				}
-				if (col.IsCreated || col.IsUpdated) && session.Statement.UseAutoTime {
+				if (col.IsCreated || col.IsUpdated) && session.statement.UseAutoTime {
-					val, t := session.Engine.NowTime2(col.SQLType.Name)
+					val, t := session.engine.nowTime(col)
 					args = append(args, val)
 
 					var colName = col.Name
@@ -188,7 +189,7 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 						col := table.GetColumn(colName)
 						setColumnTime(bean, col, t)
 					})
-				} else if col.IsVersion && session.Statement.checkVersion {
+				} else if col.IsVersion && session.statement.checkVersion {
 					args = append(args, 1)
 					var colName = col.Name
 					session.afterClosures = append(session.afterClosures, func(bean interface{}) {
@@ -212,25 +213,26 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 
 	var sql = "INSERT INTO %s (%v%v%v) VALUES (%v)"
 	var statement string
-	if session.Engine.dialect.DBType() == core.ORACLE {
+	var tableName = session.statement.TableName()
+	if session.engine.dialect.DBType() == core.ORACLE {
 		sql = "INSERT ALL INTO %s (%v%v%v) VALUES (%v) SELECT 1 FROM DUAL"
 		temp := fmt.Sprintf(") INTO %s (%v%v%v) VALUES (",
-			session.Engine.Quote(session.Statement.TableName()),
+			session.engine.Quote(tableName),
-			session.Engine.QuoteStr(),
+			session.engine.QuoteStr(),
-			strings.Join(colNames, session.Engine.QuoteStr() + ", " + session.Engine.QuoteStr()),
+			strings.Join(colNames, session.engine.QuoteStr()+", "+session.engine.QuoteStr()),
-			session.Engine.QuoteStr())
+			session.engine.QuoteStr())
 		statement = fmt.Sprintf(sql,
-			session.Engine.Quote(session.Statement.TableName()),
+			session.engine.Quote(tableName),
-			session.Engine.QuoteStr(),
+			session.engine.QuoteStr(),
-			strings.Join(colNames, session.Engine.QuoteStr() + ", " + session.Engine.QuoteStr()),
+			strings.Join(colNames, session.engine.QuoteStr()+", "+session.engine.QuoteStr()),
-			session.Engine.QuoteStr(),
+			session.engine.QuoteStr(),
 			strings.Join(colMultiPlaces, temp))
 	} else {
 		statement = fmt.Sprintf(sql,
-			session.Engine.Quote(session.Statement.TableName()),
+			session.engine.Quote(tableName),
-			session.Engine.QuoteStr(),
+			session.engine.QuoteStr(),
-			strings.Join(colNames, session.Engine.QuoteStr() + ", " + session.Engine.QuoteStr()),
+			strings.Join(colNames, session.engine.QuoteStr()+", "+session.engine.QuoteStr()),
-			session.Engine.QuoteStr(),
+			session.engine.QuoteStr(),
 			strings.Join(colMultiPlaces, "),("))
 	}
 	res, err := session.exec(statement, args...)
@@ -238,8 +240,8 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 		return 0, err
 	}
 
-	if cacher := session.Engine.getCacher2(table); cacher != nil && session.Statement.UseCache {
+	if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache {
-		session.cacheInsert(session.Statement.TableName())
+		session.cacheInsert(table, tableName)
 	}
 
 	lenAfterClosures := len(session.afterClosures)
@@ -247,7 +249,7 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 		elemValue := reflect.Indirect(sliceValue.Index(i)).Addr().Interface()
 
 		// handle AfterInsertProcessor
-		if session.IsAutoCommit {
+		if session.isAutoCommit {
 			// !nashtsai! does user expect it's same slice to passed closure when using Before()/After() when insert multi??
 			for _, closure := range session.afterClosures {
 				closure(elemValue)
@@ -278,8 +280,7 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 
 // InsertMulti insert multiple records
 func (session *Session) InsertMulti(rowsSlicePtr interface{}) (int64, error) {
-	defer session.resetStatement()
+	if session.isAutoClose {
-	if session.IsAutoClose {
 		defer session.Close()
 	}
 
@@ -297,12 +298,14 @@ func (session *Session) InsertMulti(rowsSlicePtr interface{}) (int64, error) {
 }
 
 func (session *Session) innerInsert(bean interface{}) (int64, error) {
-	session.Statement.setRefValue(rValue(bean))
+	if err := session.statement.setRefValue(rValue(bean)); err != nil {
-	if len(session.Statement.TableName()) <= 0 {
+		return 0, err
+	}
+	if len(session.statement.TableName()) <= 0 {
 		return 0, ErrTableNotFound
 	}
 
-	table := session.Statement.RefTable
+	table := session.statement.RefTable
 
 	// handle BeforeInsertProcessor
 	for _, closure := range session.beforeClosures {
@@ -314,19 +317,19 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 		processor.BeforeInsert()
 	}
 	// --
-	colNames, args, err := genCols(session.Statement.RefTable, session, bean, false, false)
+	colNames, args, err := genCols(session.statement.RefTable, session, bean, false, false)
 	if err != nil {
 		return 0, err
 	}
 	// insert expr columns, override if exists
-	exprColumns := session.Statement.getExpr()
+	exprColumns := session.statement.getExpr()
 	exprColVals := make([]string, 0, len(exprColumns))
 	for _, v := range exprColumns {
 		// remove the expr columns
 		for i, colName := range colNames {
 			if colName == v.colName {
-				colNames = append(colNames[:i], colNames[i + 1:]...)
+				colNames = append(colNames[:i], colNames[i+1:]...)
-				args = append(args[:i], args[i + 1:]...)
+				args = append(args[:i], args[i+1:]...)
 			}
 		}
 
@@ -335,22 +338,34 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 		exprColVals = append(exprColVals, v.expr)
 	}
 
-	colPlaces := strings.Repeat("?, ", len(colNames) - len(exprColumns))
+	colPlaces := strings.Repeat("?, ", len(colNames)-len(exprColumns))
 	if len(exprColVals) > 0 {
 		colPlaces = colPlaces + strings.Join(exprColVals, ", ")
 	} else {
-		colPlaces = colPlaces[0 : len(colPlaces) - 2]
+		if len(colPlaces) > 0 {
+			colPlaces = colPlaces[0 : len(colPlaces)-2]
+		}
 	}
 
-	sqlStr := fmt.Sprintf("INSERT INTO %s (%v%v%v) VALUES (%v)",
+	var sqlStr string
-		session.Engine.Quote(session.Statement.TableName()),
+	var tableName = session.statement.TableName()
-		session.Engine.QuoteStr(),
+	if len(colPlaces) > 0 {
-		strings.Join(colNames, session.Engine.Quote(", ")),
+		sqlStr = fmt.Sprintf("INSERT INTO %s (%v%v%v) VALUES (%v)",
-		session.Engine.QuoteStr(),
+			session.engine.Quote(tableName),
-		colPlaces)
+			session.engine.QuoteStr(),
+			strings.Join(colNames, session.engine.Quote(", ")),
+			session.engine.QuoteStr(),
+			colPlaces)
+	} else {
+		if session.engine.dialect.DBType() == core.MYSQL {
+			sqlStr = fmt.Sprintf("INSERT INTO %s VALUES ()", session.engine.Quote(tableName))
+		} else {
+			sqlStr = fmt.Sprintf("INSERT INTO %s DEFAULT VALUES", session.engine.Quote(tableName))
+		}
+	}
 
 	handleAfterInsertProcessorFunc := func(bean interface{}) {
-		if session.IsAutoCommit {
+		if session.isAutoCommit {
 			for _, closure := range session.afterClosures {
 				closure(bean)
 			}
@@ -379,23 +394,22 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 
 	// for postgres, many of them didn't implement lastInsertId, so we should
 	// implemented it ourself.
-	if session.Engine.dialect.DBType() == core.ORACLE && len(table.AutoIncrement) > 0 {
+	if session.engine.dialect.DBType() == core.ORACLE && len(table.AutoIncrement) > 0 {
-		//assert table.AutoIncrement != ""
+		res, err := session.queryBytes("select seq_atable.currval from dual", args...)
-		res, err := session.query("select seq_atable.currval from dual", args...)
 		if err != nil {
 			return 0, err
 		}
 
-		handleAfterInsertProcessorFunc(bean)
+		defer handleAfterInsertProcessorFunc(bean)
 
-		if cacher := session.Engine.getCacher2(table); cacher != nil && session.Statement.UseCache {
+		if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache {
-			session.cacheInsert(session.Statement.TableName())
+			session.cacheInsert(table, tableName)
 		}
 
-		if table.Version != "" && session.Statement.checkVersion {
+		if table.Version != "" && session.statement.checkVersion {
 			verValue, err := table.VersionColumn().ValueOf(bean)
 			if err != nil {
-				session.Engine.logger.Error(err)
+				session.engine.logger.Error(err)
 			} else if verValue.IsValid() && verValue.CanSet() {
 				verValue.SetInt(1)
 			}
@@ -413,7 +427,7 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 
 		aiValue, err := table.AutoIncrColumn().ValueOf(bean)
 		if err != nil {
-			session.Engine.logger.Error(err)
+			session.engine.logger.Error(err)
 		}
 
 		if aiValue == nil || !aiValue.IsValid() || !aiValue.CanSet() {
@@ -423,24 +437,24 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 		aiValue.Set(int64ToIntValue(id, aiValue.Type()))
 
 		return 1, nil
-	} else if session.Engine.dialect.DBType() == core.POSTGRES && len(table.AutoIncrement) > 0 {
+	} else if session.engine.dialect.DBType() == core.POSTGRES && len(table.AutoIncrement) > 0 {
 		//assert table.AutoIncrement != ""
-		sqlStr = sqlStr + " RETURNING " + session.Engine.Quote(table.AutoIncrement)
+		sqlStr = sqlStr + " RETURNING " + session.engine.Quote(table.AutoIncrement)
-		res, err := session.query(sqlStr, args...)
+		res, err := session.queryBytes(sqlStr, args...)
 
 		if err != nil {
 			return 0, err
 		}
-		handleAfterInsertProcessorFunc(bean)
+		defer handleAfterInsertProcessorFunc(bean)
 
-		if cacher := session.Engine.getCacher2(table); cacher != nil && session.Statement.UseCache {
+		if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache {
-			session.cacheInsert(session.Statement.TableName())
+			session.cacheInsert(table, tableName)
 		}
 
-		if table.Version != "" && session.Statement.checkVersion {
+		if table.Version != "" && session.statement.checkVersion {
 			verValue, err := table.VersionColumn().ValueOf(bean)
 			if err != nil {
-				session.Engine.logger.Error(err)
+				session.engine.logger.Error(err)
 			} else if verValue.IsValid() && verValue.CanSet() {
 				verValue.SetInt(1)
 			}
@@ -458,7 +472,7 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 
 		aiValue, err := table.AutoIncrColumn().ValueOf(bean)
 		if err != nil {
-			session.Engine.logger.Error(err)
+			session.engine.logger.Error(err)
 		}
 
 		if aiValue == nil || !aiValue.IsValid() || !aiValue.CanSet() {
@@ -476,14 +490,14 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 
 		defer handleAfterInsertProcessorFunc(bean)
 
-		if cacher := session.Engine.getCacher2(table); cacher != nil && session.Statement.UseCache {
+		if cacher := session.engine.getCacher2(table); cacher != nil && session.statement.UseCache {
-			session.cacheInsert(session.Statement.TableName())
+			session.cacheInsert(table, tableName)
 		}
 
-		if table.Version != "" && session.Statement.checkVersion {
+		if table.Version != "" && session.statement.checkVersion {
 			verValue, err := table.VersionColumn().ValueOf(bean)
 			if err != nil {
-				session.Engine.logger.Error(err)
+				session.engine.logger.Error(err)
 			} else if verValue.IsValid() && verValue.CanSet() {
 				verValue.SetInt(1)
 			}
@@ -501,7 +515,7 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 
 		aiValue, err := table.AutoIncrColumn().ValueOf(bean)
 		if err != nil {
-			session.Engine.logger.Error(err)
+			session.engine.logger.Error(err)
 		}
 
 		if aiValue == nil || !aiValue.IsValid() || !aiValue.CanSet() {
@@ -518,24 +532,21 @@ func (session *Session) innerInsert(bean interface{}) (int64, error) {
 // The in parameter bean must a struct or a point to struct. The return
 // parameter is inserted and error
 func (session *Session) InsertOne(bean interface{}) (int64, error) {
-	defer session.resetStatement()
+	if session.isAutoClose {
-	if session.IsAutoClose {
 		defer session.Close()
 	}
 
 	return session.innerInsert(bean)
 }
 
-func (session *Session) cacheInsert(tables ...string) error {
+func (session *Session) cacheInsert(table *core.Table, tables ...string) error {
-	if session.Statement.RefTable == nil {
+	if table == nil {
 		return ErrCacheFailed
 	}
 
-	table := session.Statement.RefTable
+	cacher := session.engine.getCacher2(table)
-	cacher := session.Engine.getCacher2(table)
-
 	for _, t := range tables {
-		session.Engine.logger.Debug("[cache] clear sql:", t)
+		session.engine.logger.Debug("[cache] clear sql:", t)
 		cacher.ClearIds(t)
 	}
 

vendor/github.com/go-xorm/xorm/session_iterate.go

@@ -19,6 +19,14 @@ func (session *Session) Rows(bean interface{}) (*Rows, error) {
 // are conditions. beans could be []Struct, []*Struct, map[int64]Struct
 // map[int64]*Struct
 func (session *Session) Iterate(bean interface{}, fun IterFunc) error {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	if session.statement.bufferSize > 0 {
+		return session.bufferIterate(bean, fun)
+	}
+
 	rows, err := session.Rows(bean)
 	if err != nil {
 		return err
@@ -40,3 +48,49 @@ func (session *Session) Iterate(bean interface{}, fun IterFunc) error {
 	}
 	return err
 }
+
+// BufferSize sets the buffersize for iterate
+func (session *Session) BufferSize(size int) *Session {
+	session.statement.bufferSize = size
+	return session
+}
+
+func (session *Session) bufferIterate(bean interface{}, fun IterFunc) error {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	var bufferSize = session.statement.bufferSize
+	var limit = session.statement.LimitN
+	if limit > 0 && bufferSize > limit {
+		bufferSize = limit
+	}
+	var start = session.statement.Start
+	v := rValue(bean)
+	sliceType := reflect.SliceOf(v.Type())
+	var idx = 0
+	for {
+		slice := reflect.New(sliceType)
+		if err := session.Limit(bufferSize, start).find(slice.Interface(), bean); err != nil {
+			return err
+		}
+
+		for i := 0; i < slice.Elem().Len(); i++ {
+			if err := fun(idx, slice.Elem().Index(i).Addr().Interface()); err != nil {
+				return err
+			}
+			idx++
+		}
+
+		start = start + slice.Elem().Len()
+		if limit > 0 && idx+bufferSize > limit {
+			bufferSize = limit - idx
+		}
+
+		if bufferSize <= 0 || slice.Elem().Len() < bufferSize || idx == limit {
+			break
+		}
+	}
+
+	return nil
+}

vendor/github.com/go-xorm/xorm/session_query.go

@@ -0,0 +1,252 @@
+// Copyright 2017 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package xorm
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/go-xorm/builder"
+	"github.com/go-xorm/core"
+)
+
+func (session *Session) genQuerySQL(sqlorArgs ...interface{}) (string, []interface{}, error) {
+	if len(sqlorArgs) > 0 {
+		return sqlorArgs[0].(string), sqlorArgs[1:], nil
+	}
+
+	if session.statement.RawSQL != "" {
+		return session.statement.RawSQL, session.statement.RawParams, nil
+	}
+
+	if len(session.statement.TableName()) <= 0 {
+		return "", nil, ErrTableNotFound
+	}
+
+	var columnStr = session.statement.ColumnStr
+	if len(session.statement.selectStr) > 0 {
+		columnStr = session.statement.selectStr
+	} else {
+		if session.statement.JoinStr == "" {
+			if columnStr == "" {
+				if session.statement.GroupByStr != "" {
+					columnStr = session.statement.Engine.Quote(strings.Replace(session.statement.GroupByStr, ",", session.engine.Quote(","), -1))
+				} else {
+					columnStr = session.statement.genColumnStr()
+				}
+			}
+		} else {
+			if columnStr == "" {
+				if session.statement.GroupByStr != "" {
+					columnStr = session.statement.Engine.Quote(strings.Replace(session.statement.GroupByStr, ",", session.engine.Quote(","), -1))
+				} else {
+					columnStr = "*"
+				}
+			}
+		}
+		if columnStr == "" {
+			columnStr = "*"
+		}
+	}
+
+	condSQL, condArgs, err := builder.ToSQL(session.statement.cond)
+	if err != nil {
+		return "", nil, err
+	}
+
+	args := append(session.statement.joinArgs, condArgs...)
+	sqlStr, err := session.statement.genSelectSQL(columnStr, condSQL)
+	if err != nil {
+		return "", nil, err
+	}
+	// for mssql and use limit
+	qs := strings.Count(sqlStr, "?")
+	if len(args)*2 == qs {
+		args = append(args, args...)
+	}
+
+	return sqlStr, args, nil
+}
+
+// Query runs a raw sql and return records as []map[string][]byte
+func (session *Session) Query(sqlorArgs ...interface{}) ([]map[string][]byte, error) {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	sqlStr, args, err := session.genQuerySQL(sqlorArgs...)
+	if err != nil {
+		return nil, err
+	}
+
+	return session.queryBytes(sqlStr, args...)
+}
+
+func value2String(rawValue *reflect.Value) (str string, err error) {
+	aa := reflect.TypeOf((*rawValue).Interface())
+	vv := reflect.ValueOf((*rawValue).Interface())
+	switch aa.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		str = strconv.FormatInt(vv.Int(), 10)
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		str = strconv.FormatUint(vv.Uint(), 10)
+	case reflect.Float32, reflect.Float64:
+		str = strconv.FormatFloat(vv.Float(), 'f', -1, 64)
+	case reflect.String:
+		str = vv.String()
+	case reflect.Array, reflect.Slice:
+		switch aa.Elem().Kind() {
+		case reflect.Uint8:
+			data := rawValue.Interface().([]byte)
+			str = string(data)
+			if str == "\x00" {
+				str = "0"
+			}
+		default:
+			err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
+		}
+	// time type
+	case reflect.Struct:
+		if aa.ConvertibleTo(core.TimeType) {
+			str = vv.Convert(core.TimeType).Interface().(time.Time).Format(time.RFC3339Nano)
+		} else {
+			err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
+		}
+	case reflect.Bool:
+		str = strconv.FormatBool(vv.Bool())
+	case reflect.Complex128, reflect.Complex64:
+		str = fmt.Sprintf("%v", vv.Complex())
+	/* TODO: unsupported types below
+	   case reflect.Map:
+	   case reflect.Ptr:
+	   case reflect.Uintptr:
+	   case reflect.UnsafePointer:
+	   case reflect.Chan, reflect.Func, reflect.Interface:
+	*/
+	default:
+		err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
+	}
+	return
+}
+
+func row2mapStr(rows *core.Rows, fields []string) (resultsMap map[string]string, err error) {
+	result := make(map[string]string)
+	scanResultContainers := make([]interface{}, len(fields))
+	for i := 0; i < len(fields); i++ {
+		var scanResultContainer interface{}
+		scanResultContainers[i] = &scanResultContainer
+	}
+	if err := rows.Scan(scanResultContainers...); err != nil {
+		return nil, err
+	}
+
+	for ii, key := range fields {
+		rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
+		// if row is null then as empty string
+		if rawValue.Interface() == nil {
+			result[key] = ""
+			continue
+		}
+
+		if data, err := value2String(&rawValue); err == nil {
+			result[key] = data
+		} else {
+			return nil, err
+		}
+	}
+	return result, nil
+}
+
+func rows2Strings(rows *core.Rows) (resultsSlice []map[string]string, err error) {
+	fields, err := rows.Columns()
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		result, err := row2mapStr(rows, fields)
+		if err != nil {
+			return nil, err
+		}
+		resultsSlice = append(resultsSlice, result)
+	}
+
+	return resultsSlice, nil
+}
+
+// QueryString runs a raw sql and return records as []map[string]string
+func (session *Session) QueryString(sqlorArgs ...interface{}) ([]map[string]string, error) {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	sqlStr, args, err := session.genQuerySQL(sqlorArgs...)
+	if err != nil {
+		return nil, err
+	}
+
+	rows, err := session.queryRows(sqlStr, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	return rows2Strings(rows)
+}
+
+func row2mapInterface(rows *core.Rows, fields []string) (resultsMap map[string]interface{}, err error) {
+	resultsMap = make(map[string]interface{}, len(fields))
+	scanResultContainers := make([]interface{}, len(fields))
+	for i := 0; i < len(fields); i++ {
+		var scanResultContainer interface{}
+		scanResultContainers[i] = &scanResultContainer
+	}
+	if err := rows.Scan(scanResultContainers...); err != nil {
+		return nil, err
+	}
+
+	for ii, key := range fields {
+		resultsMap[key] = reflect.Indirect(reflect.ValueOf(scanResultContainers[ii])).Interface()
+	}
+	return
+}
+
+func rows2Interfaces(rows *core.Rows) (resultsSlice []map[string]interface{}, err error) {
+	fields, err := rows.Columns()
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		result, err := row2mapInterface(rows, fields)
+		if err != nil {
+			return nil, err
+		}
+		resultsSlice = append(resultsSlice, result)
+	}
+
+	return resultsSlice, nil
+}
+
+// QueryInterface runs a raw sql and return records as []map[string]interface{}
+func (session *Session) QueryInterface(sqlorArgs ...interface{}) ([]map[string]interface{}, error) {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	sqlStr, args, err := session.genQuerySQL(sqlorArgs...)
+	if err != nil {
+		return nil, err
+	}
+
+	rows, err := session.queryRows(sqlStr, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	return rows2Interfaces(rows)
+}

vendor/github.com/go-xorm/xorm/session_raw.go

@@ -6,21 +6,140 @@ package xorm
 
 import (
 	"database/sql"
+	"reflect"
+	"time"
 
 	"github.com/go-xorm/core"
 )
 
-func (session *Session) query(sqlStr string, paramStr ...interface{}) (resultsSlice []map[string][]byte, err error) {
+func (session *Session) queryPreprocess(sqlStr *string, paramStr ...interface{}) {
-	session.queryPreprocess(&sqlStr, paramStr...)
+	for _, filter := range session.engine.dialect.Filters() {
-
+		*sqlStr = filter.Do(*sqlStr, session.engine.dialect, session.statement.RefTable)
-	if session.IsAutoCommit {
-		return session.innerQuery2(sqlStr, paramStr...)
 	}
-	return session.txQuery(session.Tx, sqlStr, paramStr...)
+
+	session.lastSQL = *sqlStr
+	session.lastSQLArgs = paramStr
 }
 
-func (session *Session) txQuery(tx *core.Tx, sqlStr string, params ...interface{}) (resultsSlice []map[string][]byte, err error) {
+func (session *Session) queryRows(sqlStr string, args ...interface{}) (*core.Rows, error) {
-	rows, err := tx.Query(sqlStr, params...)
+	defer session.resetStatement()
+
+	session.queryPreprocess(&sqlStr, args...)
+
+	if session.engine.showSQL {
+		if session.engine.showExecTime {
+			b4ExecTime := time.Now()
+			defer func() {
+				execDuration := time.Since(b4ExecTime)
+				if len(args) > 0 {
+					session.engine.logger.Infof("[SQL] %s %#v - took: %v", sqlStr, args, execDuration)
+				} else {
+					session.engine.logger.Infof("[SQL] %s - took: %v", sqlStr, execDuration)
+				}
+			}()
+		} else {
+			if len(args) > 0 {
+				session.engine.logger.Infof("[SQL] %v %#v", sqlStr, args)
+			} else {
+				session.engine.logger.Infof("[SQL] %v", sqlStr)
+			}
+		}
+	}
+
+	if session.isAutoCommit {
+		var db *core.DB
+		if session.engine.engineGroup != nil {
+			db = session.engine.engineGroup.Slave().DB()
+		} else {
+			db = session.DB()
+		}
+
+		if session.prepareStmt {
+			// don't clear stmt since session will cache them
+			stmt, err := session.doPrepare(db, sqlStr)
+			if err != nil {
+				return nil, err
+			}
+
+			rows, err := stmt.Query(args...)
+			if err != nil {
+				return nil, err
+			}
+			return rows, nil
+		}
+
+		rows, err := db.Query(sqlStr, args...)
+		if err != nil {
+			return nil, err
+		}
+		return rows, nil
+	}
+
+	rows, err := session.tx.Query(sqlStr, args...)
+	if err != nil {
+		return nil, err
+	}
+	return rows, nil
+}
+
+func (session *Session) queryRow(sqlStr string, args ...interface{}) *core.Row {
+	return core.NewRow(session.queryRows(sqlStr, args...))
+}
+
+func value2Bytes(rawValue *reflect.Value) ([]byte, error) {
+	str, err := value2String(rawValue)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(str), nil
+}
+
+func row2map(rows *core.Rows, fields []string) (resultsMap map[string][]byte, err error) {
+	result := make(map[string][]byte)
+	scanResultContainers := make([]interface{}, len(fields))
+	for i := 0; i < len(fields); i++ {
+		var scanResultContainer interface{}
+		scanResultContainers[i] = &scanResultContainer
+	}
+	if err := rows.Scan(scanResultContainers...); err != nil {
+		return nil, err
+	}
+
+	for ii, key := range fields {
+		rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
+		//if row is null then ignore
+		if rawValue.Interface() == nil {
+			result[key] = []byte{}
+			continue
+		}
+
+		if data, err := value2Bytes(&rawValue); err == nil {
+			result[key] = data
+		} else {
+			return nil, err // !nashtsai! REVIEW, should return err or just error log?
+		}
+	}
+	return result, nil
+}
+
+func rows2maps(rows *core.Rows) (resultsSlice []map[string][]byte, err error) {
+	fields, err := rows.Columns()
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		result, err := row2map(rows, fields)
+		if err != nil {
+			return nil, err
+		}
+		resultsSlice = append(resultsSlice, result)
+	}
+
+	return resultsSlice, nil
+}
+
+func (session *Session) queryBytes(sqlStr string, args ...interface{}) ([]map[string][]byte, error) {
+	rows, err := session.queryRows(sqlStr, args...)
 	if err != nil {
 		return nil, err
 	}
@@ -29,73 +148,37 @@ func (session *Session) txQuery(tx *core.Tx, sqlStr string, params ...interface{
 	return rows2maps(rows)
 }
 
-func (session *Session) innerQuery(sqlStr string, params ...interface{}) (*core.Stmt, *core.Rows, error) {
+func (session *Session) exec(sqlStr string, args ...interface{}) (sql.Result, error) {
-	var callback func() (*core.Stmt, *core.Rows, error)
-	if session.prepareStmt {
-		callback = func() (*core.Stmt, *core.Rows, error) {
-			stmt, err := session.doPrepare(sqlStr)
-			if err != nil {
-				return nil, nil, err
-			}
-			rows, err := stmt.Query(params...)
-			if err != nil {
-				return nil, nil, err
-			}
-			return stmt, rows, nil
-		}
-	} else {
-		callback = func() (*core.Stmt, *core.Rows, error) {
-			rows, err := session.DB().Query(sqlStr, params...)
-			if err != nil {
-				return nil, nil, err
-			}
-			return nil, rows, err
-		}
-	}
-	stmt, rows, err := session.Engine.logSQLQueryTime(sqlStr, params, callback)
-	if err != nil {
-		return nil, nil, err
-	}
-	return stmt, rows, nil
-}
-
-func (session *Session) innerQuery2(sqlStr string, params ...interface{}) ([]map[string][]byte, error) {
-	_, rows, err := session.innerQuery(sqlStr, params...)
-	if rows != nil {
-		defer rows.Close()
-	}
-	if err != nil {
-		return nil, err
-	}
-	return rows2maps(rows)
-}
-
-// Query a raw sql and return records as []map[string][]byte
-func (session *Session) Query(sqlStr string, paramStr ...interface{}) (resultsSlice []map[string][]byte, err error) {
 	defer session.resetStatement()
-	if session.IsAutoClose {
+
-		defer session.Close()
+	session.queryPreprocess(&sqlStr, args...)
+
+	if session.engine.showSQL {
+		if session.engine.showExecTime {
+			b4ExecTime := time.Now()
+			defer func() {
+				execDuration := time.Since(b4ExecTime)
+				if len(args) > 0 {
+					session.engine.logger.Infof("[SQL] %s %#v - took: %v", sqlStr, args, execDuration)
+				} else {
+					session.engine.logger.Infof("[SQL] %s - took: %v", sqlStr, execDuration)
+				}
+			}()
+		} else {
+			if len(args) > 0 {
+				session.engine.logger.Infof("[SQL] %v %#v", sqlStr, args)
+			} else {
+				session.engine.logger.Infof("[SQL] %v", sqlStr)
+			}
+		}
 	}
 
-	return session.query(sqlStr, paramStr...)
+	if !session.isAutoCommit {
-}
+		return session.tx.Exec(sqlStr, args...)
-
-// =============================
-// for string
-// =============================
-func (session *Session) query2(sqlStr string, paramStr ...interface{}) (resultsSlice []map[string]string, err error) {
-	session.queryPreprocess(&sqlStr, paramStr...)
-
-	if session.IsAutoCommit {
-		return query2(session.DB(), sqlStr, paramStr...)
 	}
-	return txQuery2(session.Tx, sqlStr, paramStr...)
-}
 
-// Execute sql
-func (session *Session) innerExec(sqlStr string, args ...interface{}) (sql.Result, error) {
 	if session.prepareStmt {
-		stmt, err := session.doPrepare(sqlStr)
+		stmt, err := session.doPrepare(session.DB(), sqlStr)
 		if err != nil {
 			return nil, err
 		}
@@ -110,33 +193,9 @@ func (session *Session) innerExec(sqlStr string, args ...interface{}) (sql.Resul
 	return session.DB().Exec(sqlStr, args...)
 }
 
-func (session *Session) exec(sqlStr string, args ...interface{}) (sql.Result, error) {
-	for _, filter := range session.Engine.dialect.Filters() {
-		// TODO: for table name, it's no need to RefTable
-		sqlStr = filter.Do(sqlStr, session.Engine.dialect, session.Statement.RefTable)
-	}
-
-	session.saveLastSQL(sqlStr, args...)
-
-	return session.Engine.logSQLExecutionTime(sqlStr, args, func() (sql.Result, error) {
-		if session.IsAutoCommit {
-			// FIXME: oci8 can not auto commit (github.com/mattn/go-oci8)
-			if session.Engine.dialect.DBType() == core.ORACLE {
-				session.Begin()
-				r, err := session.Tx.Exec(sqlStr, args...)
-				session.Commit()
-				return r, err
-			}
-			return session.innerExec(sqlStr, args...)
-		}
-		return session.Tx.Exec(sqlStr, args...)
-	})
-}
-
 // Exec raw sql
 func (session *Session) Exec(sqlStr string, args ...interface{}) (sql.Result, error) {
-	defer session.resetStatement()
+	if session.isAutoClose {
-	if session.IsAutoClose {
 		defer session.Close()
 	}
 

vendor/github.com/go-xorm/xorm/session_schema.go

@@ -16,38 +16,50 @@ import (
 
 // Ping test if database is ok
 func (session *Session) Ping() error {
-	defer session.resetStatement()
+	if session.isAutoClose {
-	if session.IsAutoClose {
 		defer session.Close()
 	}
 
+	session.engine.logger.Infof("PING DATABASE %v", session.engine.DriverName())
 	return session.DB().Ping()
 }
 
 // CreateTable create a table according a bean
 func (session *Session) CreateTable(bean interface{}) error {
-	v := rValue(bean)
+	if session.isAutoClose {
-	session.Statement.setRefValue(v)
-
-	defer session.resetStatement()
-	if session.IsAutoClose {
 		defer session.Close()
 	}
 
-	return session.createOneTable()
+	return session.createTable(bean)
+}
+
+func (session *Session) createTable(bean interface{}) error {
+	v := rValue(bean)
+	if err := session.statement.setRefValue(v); err != nil {
+		return err
+	}
+
+	sqlStr := session.statement.genCreateTableSQL()
+	_, err := session.exec(sqlStr)
+	return err
 }
 
 // CreateIndexes create indexes
 func (session *Session) CreateIndexes(bean interface{}) error {
-	v := rValue(bean)
+	if session.isAutoClose {
-	session.Statement.setRefValue(v)
-
-	defer session.resetStatement()
-	if session.IsAutoClose {
 		defer session.Close()
 	}
 
-	sqls := session.Statement.genIndexSQL()
+	return session.createIndexes(bean)
+}
+
+func (session *Session) createIndexes(bean interface{}) error {
+	v := rValue(bean)
+	if err := session.statement.setRefValue(v); err != nil {
+		return err
+	}
+
+	sqls := session.statement.genIndexSQL()
 	for _, sqlStr := range sqls {
 		_, err := session.exec(sqlStr)
 		if err != nil {
@@ -59,15 +71,19 @@ func (session *Session) CreateIndexes(bean interface{}) error {
 
 // CreateUniques create uniques
 func (session *Session) CreateUniques(bean interface{}) error {
-	v := rValue(bean)
+	if session.isAutoClose {
-	session.Statement.setRefValue(v)
-
-	defer session.resetStatement()
-	if session.IsAutoClose {
 		defer session.Close()
 	}
+	return session.createUniques(bean)
+}
 
-	sqls := session.Statement.genUniqueSQL()
+func (session *Session) createUniques(bean interface{}) error {
+	v := rValue(bean)
+	if err := session.statement.setRefValue(v); err != nil {
+		return err
+	}
+
+	sqls := session.statement.genUniqueSQL()
 	for _, sqlStr := range sqls {
 		_, err := session.exec(sqlStr)
 		if err != nil {
@@ -77,41 +93,22 @@ func (session *Session) CreateUniques(bean interface{}) error {
 	return nil
 }
 
-func (session *Session) createOneTable() error {
-	sqlStr := session.Statement.genCreateTableSQL()
-	_, err := session.exec(sqlStr)
-	return err
-}
-
-// to be deleted
-func (session *Session) createAll() error {
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-
-	for _, table := range session.Engine.Tables {
-		session.Statement.RefTable = table
-		session.Statement.tableName = table.Name
-		err := session.createOneTable()
-		session.resetStatement()
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // DropIndexes drop indexes
 func (session *Session) DropIndexes(bean interface{}) error {
-	v := rValue(bean)
+	if session.isAutoClose {
-	session.Statement.setRefValue(v)
-
-	defer session.resetStatement()
-	if session.IsAutoClose {
 		defer session.Close()
 	}
 
-	sqls := session.Statement.genDelIndexSQL()
+	return session.dropIndexes(bean)
+}
+
+func (session *Session) dropIndexes(bean interface{}) error {
+	v := rValue(bean)
+	if err := session.statement.setRefValue(v); err != nil {
+		return err
+	}
+
+	sqls := session.statement.genDelIndexSQL()
 	for _, sqlStr := range sqls {
 		_, err := session.exec(sqlStr)
 		if err != nil {
@@ -123,15 +120,23 @@ func (session *Session) DropIndexes(bean interface{}) error {
 
 // DropTable drop table will drop table if exist, if drop failed, it will return error
 func (session *Session) DropTable(beanOrTableName interface{}) error {
-	tableName, err := session.Engine.tableName(beanOrTableName)
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	return session.dropTable(beanOrTableName)
+}
+
+func (session *Session) dropTable(beanOrTableName interface{}) error {
+	tableName, err := session.engine.tableName(beanOrTableName)
 	if err != nil {
 		return err
 	}
 
 	var needDrop = true
-	if !session.Engine.dialect.SupportDropIfExists() {
+	if !session.engine.dialect.SupportDropIfExists() {
-		sqlStr, args := session.Engine.dialect.TableCheckSql(tableName)
+		sqlStr, args := session.engine.dialect.TableCheckSql(tableName)
-		results, err := session.query(sqlStr, args...)
+		results, err := session.queryBytes(sqlStr, args...)
 		if err != nil {
 			return err
 		}
@@ -139,7 +144,7 @@ func (session *Session) DropTable(beanOrTableName interface{}) error {
 	}
 
 	if needDrop {
-		sqlStr := session.Engine.Dialect().DropTableSql(tableName)
+		sqlStr := session.engine.Dialect().DropTableSql(tableName)
 		_, err = session.exec(sqlStr)
 		return err
 	}
@@ -148,7 +153,11 @@ func (session *Session) DropTable(beanOrTableName interface{}) error {
 
 // IsTableExist if a table is exist
 func (session *Session) IsTableExist(beanOrTableName interface{}) (bool, error) {
-	tableName, err := session.Engine.tableName(beanOrTableName)
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	tableName, err := session.engine.tableName(beanOrTableName)
 	if err != nil {
 		return false, err
 	}
@@ -157,12 +166,8 @@ func (session *Session) IsTableExist(beanOrTableName interface{}) (bool, error)
 }
 
 func (session *Session) isTableExist(tableName string) (bool, error) {
-	defer session.resetStatement()
+	sqlStr, args := session.engine.dialect.TableCheckSql(tableName)
-	if session.IsAutoClose {
+	results, err := session.queryBytes(sqlStr, args...)
-		defer session.Close()
-	}
-	sqlStr, args := session.Engine.dialect.TableCheckSql(tableName)
-	results, err := session.query(sqlStr, args...)
 	return len(results) > 0, err
 }
 
@@ -172,6 +177,9 @@ func (session *Session) IsTableEmpty(bean interface{}) (bool, error) {
 	t := v.Type()
 
 	if t.Kind() == reflect.String {
+		if session.isAutoClose {
+			defer session.Close()
+		}
 		return session.isTableEmpty(bean.(string))
 	} else if t.Kind() == reflect.Struct {
 		rows, err := session.Count(bean)
@@ -181,15 +189,9 @@ func (session *Session) IsTableEmpty(bean interface{}) (bool, error) {
 }
 
 func (session *Session) isTableEmpty(tableName string) (bool, error) {
-	defer session.resetStatement()
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-
 	var total int64
-	sqlStr := fmt.Sprintf("select count(*) from %s", session.Engine.Quote(tableName))
+	sqlStr := fmt.Sprintf("select count(*) from %s", session.engine.Quote(tableName))
-	err := session.DB().QueryRow(sqlStr).Scan(&total)
+	err := session.queryRow(sqlStr).Scan(&total)
-	session.saveLastSQL(sqlStr)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			err = nil
@@ -200,30 +202,9 @@ func (session *Session) isTableEmpty(tableName string) (bool, error) {
 	return total == 0, nil
 }
 
-func (session *Session) isIndexExist(tableName, idxName string, unique bool) (bool, error) {
-	defer session.resetStatement()
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-	var idx string
-	if unique {
-		idx = uniqueName(tableName, idxName)
-	} else {
-		idx = indexName(tableName, idxName)
-	}
-	sqlStr, args := session.Engine.dialect.IndexCheckSql(tableName, idx)
-	results, err := session.query(sqlStr, args...)
-	return len(results) > 0, err
-}
-
 // find if index is exist according cols
 func (session *Session) isIndexExist2(tableName string, cols []string, unique bool) (bool, error) {
-	defer session.resetStatement()
+	indexes, err := session.engine.dialect.GetIndexes(tableName)
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-
-	indexes, err := session.Engine.dialect.GetIndexes(tableName)
 	if err != nil {
 		return false, err
 	}
@@ -240,62 +221,34 @@ func (session *Session) isIndexExist2(tableName string, cols []string, unique bo
 }
 
 func (session *Session) addColumn(colName string) error {
-	defer session.resetStatement()
+	col := session.statement.RefTable.GetColumn(colName)
-	if session.IsAutoClose {
+	sql, args := session.statement.genAddColumnStr(col)
-		defer session.Close()
-	}
-
-	col := session.Statement.RefTable.GetColumn(colName)
-	sql, args := session.Statement.genAddColumnStr(col)
 	_, err := session.exec(sql, args...)
 	return err
 }
 
 func (session *Session) addIndex(tableName, idxName string) error {
-	defer session.resetStatement()
+	index := session.statement.RefTable.Indexes[idxName]
-	if session.IsAutoClose {
+	sqlStr := session.engine.dialect.CreateIndexSql(tableName, index)
-		defer session.Close()
-	}
-	index := session.Statement.RefTable.Indexes[idxName]
-	sqlStr := session.Engine.dialect.CreateIndexSql(tableName, index)
-
 	_, err := session.exec(sqlStr)
 	return err
 }
 
 func (session *Session) addUnique(tableName, uqeName string) error {
-	defer session.resetStatement()
+	index := session.statement.RefTable.Indexes[uqeName]
-	if session.IsAutoClose {
+	sqlStr := session.engine.dialect.CreateIndexSql(tableName, index)
-		defer session.Close()
-	}
-	index := session.Statement.RefTable.Indexes[uqeName]
-	sqlStr := session.Engine.dialect.CreateIndexSql(tableName, index)
 	_, err := session.exec(sqlStr)
 	return err
 }
 
-// To be deleted
-func (session *Session) dropAll() error {
-	defer session.resetStatement()
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-
-	for _, table := range session.Engine.Tables {
-		session.Statement.Init()
-		session.Statement.RefTable = table
-		sqlStr := session.Engine.Dialect().DropTableSql(session.Statement.TableName())
-		_, err := session.exec(sqlStr)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // Sync2 synchronize structs to database tables
 func (session *Session) Sync2(beans ...interface{}) error {
-	engine := session.Engine
+	engine := session.engine
+
+	if session.isAutoClose {
+		session.isAutoClose = false
+		defer session.Close()
+	}
 
 	tables, err := engine.DBMetas()
 	if err != nil {
@@ -322,17 +275,17 @@ func (session *Session) Sync2(beans ...interface{}) error {
 		}
 
 		if oriTable == nil {
-			err = session.StoreEngine(session.Statement.StoreEngine).CreateTable(bean)
+			err = session.StoreEngine(session.statement.StoreEngine).createTable(bean)
 			if err != nil {
 				return err
 			}
 
-			err = session.CreateUniques(bean)
+			err = session.createUniques(bean)
 			if err != nil {
 				return err
 			}
 
-			err = session.CreateIndexes(bean)
+			err = session.createIndexes(bean)
 			if err != nil {
 				return err
 			}
@@ -357,7 +310,7 @@ func (session *Session) Sync2(beans ...interface{}) error {
 								engine.dialect.DBType() == core.POSTGRES {
 								engine.logger.Infof("Table %s column %s change type from %s to %s\n",
 									tbName, col.Name, curType, expectedType)
-								_, err = engine.Exec(engine.dialect.ModifyColumnSql(table.Name, col))
+								_, err = session.exec(engine.dialect.ModifyColumnSql(table.Name, col))
 							} else {
 								engine.logger.Warnf("Table %s column %s db type is %s, struct type is %s\n",
 									tbName, col.Name, curType, expectedType)
@@ -367,7 +320,7 @@ func (session *Session) Sync2(beans ...interface{}) error {
 								if oriCol.Length < col.Length {
 									engine.logger.Infof("Table %s column %s change type from varchar(%d) to varchar(%d)\n",
 										tbName, col.Name, oriCol.Length, col.Length)
-									_, err = engine.Exec(engine.dialect.ModifyColumnSql(table.Name, col))
+									_, err = session.exec(engine.dialect.ModifyColumnSql(table.Name, col))
 								}
 							}
 						} else {
@@ -381,7 +334,7 @@ func (session *Session) Sync2(beans ...interface{}) error {
 							if oriCol.Length < col.Length {
 								engine.logger.Infof("Table %s column %s change type from varchar(%d) to varchar(%d)\n",
 									tbName, col.Name, oriCol.Length, col.Length)
-								_, err = engine.Exec(engine.dialect.ModifyColumnSql(table.Name, col))
+								_, err = session.exec(engine.dialect.ModifyColumnSql(table.Name, col))
 							}
 						}
 					}
@@ -394,10 +347,8 @@ func (session *Session) Sync2(beans ...interface{}) error {
 							tbName, col.Name, oriCol.Nullable, col.Nullable)
 					}
 				} else {
-					session := engine.NewSession()
+					session.statement.RefTable = table
-					session.Statement.RefTable = table
+					session.statement.tableName = tbName
-					session.Statement.tableName = tbName
-					defer session.Close()
 					err = session.addColumn(col.Name)
 				}
 				if err != nil {
@@ -421,7 +372,7 @@ func (session *Session) Sync2(beans ...interface{}) error {
 				if oriIndex != nil {
 					if oriIndex.Type != index.Type {
 						sql := engine.dialect.DropIndexSql(tbName, oriIndex)
-						_, err = engine.Exec(sql)
+						_, err = session.exec(sql)
 						if err != nil {
 							return err
 						}
@@ -437,7 +388,7 @@ func (session *Session) Sync2(beans ...interface{}) error {
 			for name2, index2 := range oriTable.Indexes {
 				if _, ok := foundIndexNames[name2]; !ok {
 					sql := engine.dialect.DropIndexSql(tbName, index2)
-					_, err = engine.Exec(sql)
+					_, err = session.exec(sql)
 					if err != nil {
 						return err
 					}
@@ -446,16 +397,12 @@ func (session *Session) Sync2(beans ...interface{}) error {
 
 			for name, index := range addedNames {
 				if index.Type == core.UniqueType {
-					session := engine.NewSession()
+					session.statement.RefTable = table
-					session.Statement.RefTable = table
+					session.statement.tableName = tbName
-					session.Statement.tableName = tbName
-					defer session.Close()
 					err = session.addUnique(tbName, name)
 				} else if index.Type == core.IndexType {
-					session := engine.NewSession()
+					session.statement.RefTable = table
-					session.Statement.RefTable = table
+					session.statement.tableName = tbName
-					session.Statement.tableName = tbName
-					defer session.Close()
 					err = session.addIndex(tbName, name)
 				}
 				if err != nil {

vendor/github.com/go-xorm/xorm/session_stats.go

@@ -0,0 +1,98 @@
+// Copyright 2016 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package xorm
+
+import (
+	"database/sql"
+	"errors"
+	"reflect"
+)
+
+// Count counts the records. bean's non-empty fields
+// are conditions.
+func (session *Session) Count(bean ...interface{}) (int64, error) {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	var sqlStr string
+	var args []interface{}
+	var err error
+	if session.statement.RawSQL == "" {
+		sqlStr, args, err = session.statement.genCountSQL(bean...)
+		if err != nil {
+			return 0, err
+		}
+	} else {
+		sqlStr = session.statement.RawSQL
+		args = session.statement.RawParams
+	}
+
+	var total int64
+	err = session.queryRow(sqlStr, args...).Scan(&total)
+	if err == sql.ErrNoRows || err == nil {
+		return total, nil
+	}
+
+	return 0, err
+}
+
+// sum call sum some column. bean's non-empty fields are conditions.
+func (session *Session) sum(res interface{}, bean interface{}, columnNames ...string) error {
+	if session.isAutoClose {
+		defer session.Close()
+	}
+
+	v := reflect.ValueOf(res)
+	if v.Kind() != reflect.Ptr {
+		return errors.New("need a pointer to a variable")
+	}
+
+	var isSlice = v.Elem().Kind() == reflect.Slice
+	var sqlStr string
+	var args []interface{}
+	var err error
+	if len(session.statement.RawSQL) == 0 {
+		sqlStr, args, err = session.statement.genSumSQL(bean, columnNames...)
+		if err != nil {
+			return err
+		}
+	} else {
+		sqlStr = session.statement.RawSQL
+		args = session.statement.RawParams
+	}
+
+	if isSlice {
+		err = session.queryRow(sqlStr, args...).ScanSlice(res)
+	} else {
+		err = session.queryRow(sqlStr, args...).Scan(res)
+	}
+	if err == sql.ErrNoRows || err == nil {
+		return nil
+	}
+	return err
+}
+
+// Sum call sum some column. bean's non-empty fields are conditions.
+func (session *Session) Sum(bean interface{}, columnName string) (res float64, err error) {
+	return res, session.sum(&res, bean, columnName)
+}
+
+// SumInt call sum some column. bean's non-empty fields are conditions.
+func (session *Session) SumInt(bean interface{}, columnName string) (res int64, err error) {
+	return res, session.sum(&res, bean, columnName)
+}
+
+// Sums call sum some columns. bean's non-empty fields are conditions.
+func (session *Session) Sums(bean interface{}, columnNames ...string) ([]float64, error) {
+	var res = make([]float64, len(columnNames), len(columnNames))
+	return res, session.sum(&res, bean, columnNames...)
+}
+
+// SumsInt sum specify columns and return as []int64 instead of []float64
+func (session *Session) SumsInt(bean interface{}, columnNames ...string) ([]int64, error) {
+	var res = make([]int64, len(columnNames), len(columnNames))
+	return res, session.sum(&res, bean, columnNames...)
+}

vendor/github.com/go-xorm/xorm/session_sum.go

@@ -1,137 +0,0 @@
-// Copyright 2016 The Xorm Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package xorm
-
-import "database/sql"
-
-// Count counts the records. bean's non-empty fields
-// are conditions.
-func (session *Session) Count(bean interface{}) (int64, error) {
-	defer session.resetStatement()
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-
-	var sqlStr string
-	var args []interface{}
-	if session.Statement.RawSQL == "" {
-		sqlStr, args = session.Statement.genCountSQL(bean)
-	} else {
-		sqlStr = session.Statement.RawSQL
-		args = session.Statement.RawParams
-	}
-
-	session.queryPreprocess(&sqlStr, args...)
-
-	var err error
-	var total int64
-	if session.IsAutoCommit {
-		err = session.DB().QueryRow(sqlStr, args...).Scan(&total)
-	} else {
-		err = session.Tx.QueryRow(sqlStr, args...).Scan(&total)
-	}
-
-	if err == sql.ErrNoRows || err == nil {
-		return total, nil
-	}
-
-	return 0, err
-}
-
-// Sum call sum some column. bean's non-empty fields are conditions.
-func (session *Session) Sum(bean interface{}, columnName string) (float64, error) {
-	defer session.resetStatement()
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-
-	var sqlStr string
-	var args []interface{}
-	if len(session.Statement.RawSQL) == 0 {
-		sqlStr, args = session.Statement.genSumSQL(bean, columnName)
-	} else {
-		sqlStr = session.Statement.RawSQL
-		args = session.Statement.RawParams
-	}
-
-	session.queryPreprocess(&sqlStr, args...)
-
-	var err error
-	var res float64
-	if session.IsAutoCommit {
-		err = session.DB().QueryRow(sqlStr, args...).Scan(&res)
-	} else {
-		err = session.Tx.QueryRow(sqlStr, args...).Scan(&res)
-	}
-
-	if err == sql.ErrNoRows || err == nil {
-		return res, nil
-	}
-	return 0, err
-}
-
-// Sums call sum some columns. bean's non-empty fields are conditions.
-func (session *Session) Sums(bean interface{}, columnNames ...string) ([]float64, error) {
-	defer session.resetStatement()
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-
-	var sqlStr string
-	var args []interface{}
-	if len(session.Statement.RawSQL) == 0 {
-		sqlStr, args = session.Statement.genSumSQL(bean, columnNames...)
-	} else {
-		sqlStr = session.Statement.RawSQL
-		args = session.Statement.RawParams
-	}
-
-	session.queryPreprocess(&sqlStr, args...)
-
-	var err error
-	var res = make([]float64, len(columnNames), len(columnNames))
-	if session.IsAutoCommit {
-		err = session.DB().QueryRow(sqlStr, args...).ScanSlice(&res)
-	} else {
-		err = session.Tx.QueryRow(sqlStr, args...).ScanSlice(&res)
-	}
-
-	if err == sql.ErrNoRows || err == nil {
-		return res, nil
-	}
-	return nil, err
-}
-
-// SumsInt sum specify columns and return as []int64 instead of []float64
-func (session *Session) SumsInt(bean interface{}, columnNames ...string) ([]int64, error) {
-	defer session.resetStatement()
-	if session.IsAutoClose {
-		defer session.Close()
-	}
-
-	var sqlStr string
-	var args []interface{}
-	if len(session.Statement.RawSQL) == 0 {
-		sqlStr, args = session.Statement.genSumSQL(bean, columnNames...)
-	} else {
-		sqlStr = session.Statement.RawSQL
-		args = session.Statement.RawParams
-	}
-
-	session.queryPreprocess(&sqlStr, args...)
-
-	var err error
-	var res = make([]int64, len(columnNames), len(columnNames))
-	if session.IsAutoCommit {
-		err = session.DB().QueryRow(sqlStr, args...).ScanSlice(&res)
-	} else {
-		err = session.Tx.QueryRow(sqlStr, args...).ScanSlice(&res)
-	}
-
-	if err == sql.ErrNoRows || err == nil {
-		return res, nil
-	}
-	return nil, err
-}

vendor/github.com/go-xorm/xorm/session_tx.go

@@ -6,14 +6,14 @@ package xorm
 
 // Begin a transaction
 func (session *Session) Begin() error {
-	if session.IsAutoCommit {
+	if session.isAutoCommit {
 		tx, err := session.DB().Begin()
 		if err != nil {
 			return err
 		}
-		session.IsAutoCommit = false
+		session.isAutoCommit = false
-		session.IsCommitedOrRollbacked = false
+		session.isCommitedOrRollbacked = false
-		session.Tx = tx
+		session.tx = tx
 		session.saveLastSQL("BEGIN TRANSACTION")
 	}
 	return nil
@@ -21,25 +21,23 @@ func (session *Session) Begin() error {
 
 // Rollback When using transaction, you can rollback if any error
 func (session *Session) Rollback() error {
-	if !session.IsAutoCommit && !session.IsCommitedOrRollbacked {
+	if !session.isAutoCommit && !session.isCommitedOrRollbacked {
-		session.saveLastSQL(session.Engine.dialect.RollBackStr())
+		session.saveLastSQL(session.engine.dialect.RollBackStr())
-		session.IsCommitedOrRollbacked = true
+		session.isCommitedOrRollbacked = true
-		return session.Tx.Rollback()
+		return session.tx.Rollback()
 	}
 	return nil
 }
 
 // Commit When using transaction, Commit will commit all operations.
 func (session *Session) Commit() error {
-	if !session.IsAutoCommit && !session.IsCommitedOrRollbacked {
+	if !session.isAutoCommit && !session.isCommitedOrRollbacked {
 		session.saveLastSQL("COMMIT")
-		session.IsCommitedOrRollbacked = true
+		session.isCommitedOrRollbacked = true
 		var err error
-		if err = session.Tx.Commit(); err == nil {
+		if err = session.tx.Commit(); err == nil {
 			// handle processors after tx committed
-
 			closureCallFunc := func(closuresPtr *[]func(interface{}), bean interface{}) {
-
 				if closuresPtr != nil {
 					for _, closure := range *closuresPtr {
 						closure(bean)