Store time as text by default to avoid problem with column affinity inconsistency.

README.md

@@ -29,7 +29,6 @@ $ cp ~/Downloads/sqlite-amalgamation-xxx/sqlite3.{c,h} $GOPATH/src/github.com/gw
 
 ### Features (not supported by database/sql/driver):
 
-* Dynamic type: currently, the SQLite3 manifest typing is respected. There is no use of the column declared type to guess the target/go type when scanning. On your side, you should try to not break column affinity rules (such as declaring a column with TIMESTAMP type (NUMERIC affinity) storing values with '2006-01-02T15:04:05.999Z07:00' format (TEXT type))...
 * Named bind parameters.
 * Partial scan: scan values may be partially scanned (by index or name) or skipped/ignored by passing nil pointer(s).
 * Null value: by default, empty string and zero time are bound to null for prepared statement's parameters (no need for NullString, NullTime but still supported).

date.go

@@ -38,7 +38,7 @@ func JulianDay(t time.Time) float64 {
 	return ns/dayInSeconds + julianDay
 }
 
-// UnixTime is an alias used to persist time as int64 (max precision is 1s and timezone is lost) (default)
+// UnixTime is an alias used to persist time as int64 (max precision is 1s and timezone is lost)
 type UnixTime time.Time
 
 // Scan implements the database/sql/Scanner interface.

date_test.go

@@ -5,10 +5,11 @@
 package sqlite_test
 
 import (
-	"github.com/bmizerany/assert"
-	. "github.com/gwenn/gosqlite"
 	"testing"
 	"time"
+
+	"github.com/bmizerany/assert"
+	. "github.com/gwenn/gosqlite"
 )
 
 func TestJulianDay(t *testing.T) {
@@ -27,7 +28,8 @@ func TestBindTime(t *testing.T) {
 	db := open(t)
 	defer checkClose(db, t)
 	var delta int
-	err := db.OneValue("SELECT CAST(strftime('%s', 'now') AS NUMERIC) - ?", &delta, time.Now())
+	//err := db.OneValue("SELECT CAST(strftime('%s', 'now') AS NUMERIC) - ?", &delta, time.Now())
+	err := db.OneValue("SELECT datetime('now') - datetime(?)", &delta, time.Now())
 	checkNoError(t, err, "Error reading date: %#v")
 	if delta != 0 {
 		t.Errorf("Delta between Go and SQLite timestamps: %d", delta)

meta.go

@@ -29,6 +29,7 @@ import "C"
 
 import (
 	"fmt"
+	"strings"
 	"unsafe"
 )
 
@@ -174,13 +175,60 @@ func (s *Stmt) ColumnOriginName(index int) string {
 }
 
 // ColumnDeclaredType returns the declared type of the table column of a particular result column in SELECT statement.
-// If the result column is an expression or subquery, then a NULL pointer is returned.
+// If the result column is an expression or subquery, then an empty string is returned.
 // The left-most column is column 0.
 // (See http://www.sqlite.org/c3ref/column_decltype.html)
 func (s *Stmt) ColumnDeclaredType(index int) string {
 	return C.GoString(C.sqlite3_column_decltype(s.stmt, C.int(index)))
 }
 
+// SQLite column type affinity
+type Affinity string
+
+const (
+	Integral  = Affinity("INTEGER")
+	Real      = Affinity("REAL")
+	Numerical = Affinity("NUMERIC")
+	None      = Affinity("NONE")
+	Textual   = Affinity("TEXT")
+)
+
+// ColumnTypeAffinity returns the type affinity of the table column of a particular result column in SELECT statement.
+// If the result column is an expression or subquery, then None is returned.
+// The left-most column is column 0.
+// (See http://sqlite.org/datatype3.html)
+func (s *Stmt) ColumnTypeAffinity(index int) Affinity {
+	if s.affinities == nil {
+		count := s.ColumnCount()
+		s.affinities = make([]Affinity, count)
+	} else {
+		if affinity := s.affinities[index]; affinity != "" {
+			return affinity
+		}
+	}
+	declType := s.ColumnDeclaredType(index)
+	if declType == "" {
+		s.affinities[index] = None
+		return None
+	}
+	declType = strings.ToUpper(declType)
+	if strings.Contains(declType, "INT") {
+		s.affinities[index] = Integral
+		return Integral
+	} else if strings.Contains(declType, "TEXT") || strings.Contains(declType, "CHAR") || strings.Contains(declType, "CLOB") {
+		s.affinities[index] = Textual
+		return Textual
+	} else if strings.Contains(declType, "BLOB") {
+		s.affinities[index] = None
+		return None
+	} else if strings.Contains(declType, "REAL") || strings.Contains(declType, "FLOA") || strings.Contains(declType, "DOUB") {
+		s.affinities[index] = Real
+		return Real
+	}
+	s.affinities[index] = Numerical
+	return Numerical
+}
+
 // ForeignKey is the description of one table's foreign key
 // See Conn.ForeignKeys
 type ForeignKey struct {

meta_test.go

@@ -5,9 +5,10 @@
 package sqlite_test
 
 import (
+	"testing"
+
 	"github.com/bmizerany/assert"
 	. "github.com/gwenn/gosqlite"
-	"testing"
 )
 
 func createIndex(db *Conn, t *testing.T) {
@@ -134,4 +135,6 @@ func TestColumnMetadata(t *testing.T) {
 	assert.Equal(t, "name", originName, "origin name")
 	declType := s.ColumnDeclaredType(0)
 	assert.Equal(t, "text", declType, "declared type")
+	affinity := s.ColumnTypeAffinity(0)
+	assert.Equal(t, Textual, affinity, "affinity")
 }

stmt.go

@@ -74,6 +74,9 @@ func (s *Stmt) specificError(msg string, a ...interface{}) error {
 	return &StmtError{ConnError{c: s.c, code: ErrSpecific, msg: fmt.Sprintf(msg, a...)}, s}
 }
 
+// CheckTypeMismatch enables type check in Scan methods (default true)
+var CheckTypeMismatch bool = true
+
 // SQL statement
 // (See http://sqlite.org/c3ref/stmt.html)
 type Stmt struct {
@@ -85,8 +88,7 @@ type Stmt struct {
 	cols               map[string]int // cached columns index by name
 	bindParameterCount int
 	params             map[string]int // cached parameter index by name
-	// Enable type check in Scan methods (default true)
-	CheckTypeMismatch bool
+	affinities         []Affinity     // cached columns type affinity
 	// Tell if the stmt should be cached (default true)
 	Cacheable bool
 }
@@ -108,7 +110,7 @@ func (c *Conn) prepare(cmd string, args ...interface{}) (*Stmt, error) {
 	if tail != nil && C.strlen(tail) > 0 {
 		t = C.GoString(tail)
 	}
-	s := &Stmt{c: c, stmt: stmt, tail: t, columnCount: -1, bindParameterCount: -1, CheckTypeMismatch: true}
+	s := &Stmt{c: c, stmt: stmt, tail: t, columnCount: -1, bindParameterCount: -1}
 	if len(args) > 0 {
 		err := s.Bind(args...)
 		if err != nil {
@@ -337,6 +339,10 @@ var NullIfEmptyString = true
 // NullIfZeroTime transforms zero time (time.Time.IsZero) to null when true (true by default)
 var NullIfZeroTime = true
 
+// DefaultTimeLayout specifies the layout used to persist time ("2006-01-02 15:04:05.999Z07:00" by default).
+// Using type alias implementing the Scanner/Valuer interfaces is suggested...
+var DefaultTimeLayout = "2006-01-02 15:04:05.999Z07:00"
+
 // BindByIndex binds value to the specified host parameter of the prepared statement.
 // Value's type/kind is used to find the storage class.
 // The leftmost SQL parameter has an index of 1.
@@ -381,7 +387,9 @@ func (s *Stmt) BindByIndex(index int, value interface{}) error {
 		if NullIfZeroTime && value.IsZero() {
 			rv = C.sqlite3_bind_null(s.stmt, i)
 		} else {
-			rv = C.sqlite3_bind_int64(s.stmt, i, C.sqlite3_int64(value.Unix()))
+			//rv = C.sqlite3_bind_int64(s.stmt, i, C.sqlite3_int64(value.Unix()))
+			cs, l := cstring(value.Format(DefaultTimeLayout))
+			rv = C.my_bind_text(s.stmt, i, cs, l)
 		}
 	case ZeroBlobLength:
 		rv = C.sqlite3_bind_zeroblob(s.stmt, i, C.int(value))
@@ -815,6 +823,9 @@ func (s *Stmt) ScanReflect(index int, v interface{}) (bool, error) {
 	return isNull, err
 }
 
+// ScanNumericalAsTime tells the driver to try to parse column with NUMERIC affinity as time.Time (using the DefaultTimeLayout)
+var ScanNumericalAsTime = false
+
 // ScanValue scans result value from a query.
 // The leftmost column/index is number 0.
 //
@@ -832,7 +843,16 @@ func (s *Stmt) ScanValue(index int, blob bool) (interface{}, bool) {
 	switch s.ColumnType(index) {
 	case Null:
 		return nil, true
-	case Text:
+	case Text: // does not work as expected if column type affinity is TEXT but inserted value was a numeric
+		if ScanNumericalAsTime && s.ColumnTypeAffinity(index) == Numerical {
+			p := C.sqlite3_column_text(s.stmt, C.int(index))
+			txt := C.GoString((*C.char)(unsafe.Pointer(p)))
+			value, err := time.Parse(DefaultTimeLayout, txt)
+			if err == nil {
+				return value, false
+			}
+			Log(-1, err.Error())
+		}
 		if blob {
 			p := C.sqlite3_column_blob(s.stmt, C.int(index))
 			n := C.sqlite3_column_bytes(s.stmt, C.int(index))
@@ -842,7 +862,7 @@ func (s *Stmt) ScanValue(index int, blob bool) (interface{}, bool) {
 		return C.GoString((*C.char)(unsafe.Pointer(p))), false
 	case Integer:
 		return int64(C.sqlite3_column_int64(s.stmt, C.int(index))), false
-	case Float:
+	case Float: // does not work as expected if column type affinity is REAL but inserted value was an integer
 		return float64(C.sqlite3_column_double(s.stmt, C.int(index))), false
 	case Blob:
 		p := C.sqlite3_column_blob(s.stmt, C.int(index))
@@ -884,7 +904,7 @@ func (s *Stmt) ScanInt(index int) (value int, isNull bool, err error) {
 	if ctype == Null {
 		isNull = true
 	} else {
-		if s.CheckTypeMismatch {
+		if CheckTypeMismatch {
 			err = s.checkTypeMismatch(ctype, Integer)
 		}
 		if i64 {
@@ -906,7 +926,7 @@ func (s *Stmt) ScanInt32(index int) (value int32, isNull bool, err error) {
 	if ctype == Null {
 		isNull = true
 	} else {
-		if s.CheckTypeMismatch {
+		if CheckTypeMismatch {
 			err = s.checkTypeMismatch(ctype, Integer)
 		}
 		value = int32(C.sqlite3_column_int(s.stmt, C.int(index)))
@@ -923,7 +943,7 @@ func (s *Stmt) ScanInt64(index int) (value int64, isNull bool, err error) {
 	if ctype == Null {
 		isNull = true
 	} else {
-		if s.CheckTypeMismatch {
+		if CheckTypeMismatch {
 			err = s.checkTypeMismatch(ctype, Integer)
 		}
 		value = int64(C.sqlite3_column_int64(s.stmt, C.int(index)))
@@ -940,7 +960,7 @@ func (s *Stmt) ScanByte(index int) (value byte, isNull bool, err error) {
 	if ctype == Null {
 		isNull = true
 	} else {
-		if s.CheckTypeMismatch {
+		if CheckTypeMismatch {
 			err = s.checkTypeMismatch(ctype, Integer)
 		}
 		value = byte(C.sqlite3_column_int(s.stmt, C.int(index)))
@@ -957,7 +977,7 @@ func (s *Stmt) ScanBool(index int) (value bool, isNull bool, err error) {
 	if ctype == Null {
 		isNull = true
 	} else {
-		if s.CheckTypeMismatch {
+		if CheckTypeMismatch {
 			err = s.checkTypeMismatch(ctype, Integer)
 		}
 		value = C.sqlite3_column_int(s.stmt, C.int(index)) == 1
@@ -974,7 +994,7 @@ func (s *Stmt) ScanDouble(index int) (value float64, isNull bool, err error) {
 	if ctype == Null {
 		isNull = true
 	} else {
-		if s.CheckTypeMismatch {
+		if CheckTypeMismatch {
 			err = s.checkTypeMismatch(ctype, Float)
 		}
 		value = float64(C.sqlite3_column_double(s.stmt, C.int(index)))
@@ -1003,11 +1023,12 @@ func (s *Stmt) ScanBlob(index int) (value []byte, isNull bool) {
 // If time is persisted as numeric, local is used.
 // The leftmost column/index is number 0.
 // Returns true when column is null.
+// The column type affinity must be consistent with the format used (INTEGER or NUMERIC or NONE for unix time, REAL or NONE for julian day).
 func (s *Stmt) ScanTime(index int) (value time.Time, isNull bool, err error) {
 	switch s.ColumnType(index) {
 	case Null:
 		isNull = true
-	case Text:
+	case Text: // does not work as expected if column type affinity is TEXT but inserted value was a numeric
 		p := C.sqlite3_column_text(s.stmt, C.int(index))
 		txt := C.GoString((*C.char)(unsafe.Pointer(p)))
 		var layout string
@@ -1049,7 +1070,7 @@ func (s *Stmt) ScanTime(index int) (value time.Time, isNull bool, err error) {
 	case Integer:
 		unixepoch := int64(C.sqlite3_column_int64(s.stmt, C.int(index)))
 		value = time.Unix(unixepoch, 0) // local time
-	case Float:
+	case Float: // does not work as expected if column affinity is REAL but inserted value was an integer
 		jd := float64(C.sqlite3_column_double(s.stmt, C.int(index)))
 		value = JulianDayToLocalTime(jd) // local time
 	default:
@@ -1063,16 +1084,16 @@ func (s *Stmt) checkTypeMismatch(source, target Type) error {
 	switch target {
 	case Integer:
 		switch source {
-		case Float:
+		case Float: // does not work if column type affinity is REAL but inserted value was an integer
 			fallthrough
-		case Text:
+		case Text: // does not work if column type affinity is TEXT but inserted value was an integer
 			fallthrough
 		case Blob:
 			return s.specificError("type mismatch, source %q vs target %q", source, target)
 		}
 	case Float:
 		switch source {
-		case Text:
+		case Text: // does not work if column type affinity is TEXT but inserted value was a real
 			fallthrough
 		case Blob:
 			return s.specificError("type mismatch, source %q vs target %q", source, target)

stmt_test.go

@@ -475,9 +475,11 @@ func TestColumnType(t *testing.T) {
 	checkNoError(t, err, "prepare error: %s")
 	defer checkFinalize(s, t)
 
+	expectedAffinities := []Affinity{Integral, Real, Integral, Textual}
 	for col := 0; col < s.ColumnCount(); col++ {
 		//println(col, s.ColumnName(col), s.ColumnOriginName(col), s.ColumnType(col), s.ColumnDeclaredType(col))
 		assert.Equal(t, Null, s.ColumnType(col), "column type")
+		assert.Equal(t, expectedAffinities[col], s.ColumnTypeAffinity(col), "column type affinity")
 	}
 }