Fix toRational overflow by calculating internally with int64s (fixes #4194)

This commit is contained in:
def
2016-05-22 18:09:55 +02:00
parent 6dd0887009
commit 834ad57d94

View File

@@ -41,26 +41,26 @@ proc toRational*[T:SomeInteger](x: T): Rational[T] =
proc toRationalSub(x: float, n: int): Rational[int] =
var
a = 0
b, c, d = 1
a = 0'i64
b, c, d = 1'i64
result = 0 // 1 # rational 0
while b <= n and d <= n:
let ac = (a+c)
let bd = (b+d)
# scale by 1000 so not overflow for high precision
let mediant = (ac/1000) / (bd/1000)
let mediant = (ac.float/1000) / (bd.float/1000)
if x == mediant:
if bd <= n:
result.num = ac
result.den = bd
result.num = ac.int
result.den = bd.int
return result
elif d > b:
result.num = c
result.den = d
result.num = c.int
result.den = d.int
return result
else:
result.num = a
result.den = b
result.num = a.int
result.den = b.int
return result
elif x > mediant:
a = ac
@@ -69,8 +69,8 @@ proc toRationalSub(x: float, n: int): Rational[int] =
c = ac
d = bd
if (b > n):
return initRational(c, d)
return initRational(a, b)
return initRational(c.int, d.int)
return initRational(a.int, b.int)
proc toRational*(x: float, n: int = high(int)): Rational[int] =
## Calculate the best rational numerator and denominator