diff --git a/lexer.go b/lexer.go
index 25635b4..813f21a 100644
--- a/lexer.go
+++ b/lexer.go
@@ -184,6 +184,8 @@
 				return l.lexVoid
 			}
 			return l.lexRvalue
+		case '_':
+			return l.errorf("cannot start number with underscore")
 		}
 
 		if l.follow("true") {
@@ -550,7 +552,6 @@
 		} else if isDigit(next) {
 			digitSeen = true
 		} else if next == '_' {
-			l.pos++
 		} else {
 			l.backup()
 			break
diff --git a/lexer_test.go b/lexer_test.go
index df2cac9..1964a57 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -618,3 +618,12 @@
 		token{Position{1, 1}, tokenError, "keys cannot contain new lines"},
 	})
 }
+
+func TestInvalidFloat(t *testing.T) {
+	testFlow(t, "a=7e1_", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 2}, tokenEqual, "="},
+		token{Position{1, 3}, tokenFloat, "7e1_"},
+		token{Position{1, 7}, tokenEOF, ""},
+	})
+}
diff --git a/parser.go b/parser.go
index ba95fa5..39f72f3 100644
--- a/parser.go
+++ b/parser.go
@@ -5,6 +5,7 @@
 import (
 	"fmt"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -211,6 +212,16 @@
 	return p.parseStart
 }
 
+var numberUnderscoreInvalidRegexp *regexp.Regexp
+
+func cleanupNumberToken(value string) (string, error) {
+	if numberUnderscoreInvalidRegexp.MatchString(value) {
+		return "", fmt.Errorf("invalid use of _ in number")
+	}
+	cleanedVal := strings.Replace(value, "_", "", -1)
+	return cleanedVal, nil
+}
+
 func (p *tomlParser) parseRvalue() interface{} {
 	tok := p.getToken()
 	if tok == nil || tok.typ == tokenEOF {
@@ -225,14 +236,20 @@
 	case tokenFalse:
 		return false
 	case tokenInteger:
-		cleanedVal := strings.Replace(tok.val, "_", "", -1)
+		cleanedVal, err := cleanupNumberToken(tok.val)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
+		}
 		val, err := strconv.ParseInt(cleanedVal, 10, 64)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
 		}
 		return val
 	case tokenFloat:
-		cleanedVal := strings.Replace(tok.val, "_", "", -1)
+		cleanedVal, err := cleanupNumberToken(tok.val)
+		if err != nil {
+			p.raiseError(tok, "%s", err)
+		}
 		val, err := strconv.ParseFloat(cleanedVal, 64)
 		if err != nil {
 			p.raiseError(tok, "%s", err)
@@ -361,3 +378,7 @@
 	parser.run()
 	return result
 }
+
+func init() {
+	numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d]|_$|^_)`)
+}
diff --git a/parser_test.go b/parser_test.go
index 4481e67..53cfcde 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -633,3 +633,25 @@
 		t.Error("Bad error message:", err.Error())
 	}
 }
+
+func TestInvalidFloatParsing(t *testing.T) {
+	_, err := Load("a=1e_2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1e2_")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=1__2")
+	if err.Error() != "(1, 3): invalid use of _ in number" {
+		t.Error("Bad error message:", err.Error())
+	}
+
+	_, err = Load("a=_1_2")
+	if err.Error() != "(1, 3): cannot start number with underscore" {
+		t.Error("Bad error message:", err.Error())
+	}
+}