Module:Math: Difference between revisions
problem with lost precision
m>Dragons flight m (precision truncation issue) |
m>Dragons flight (problem with lost precision) |
||
| Line 109: | Line 109: | ||
local lang = mw.getContentLanguage(); | local lang = mw.getContentLanguage(); | ||
local | local value_string = ( frame.args[1] or '0' ) | ||
local value = tonumber( value_string ); | |||
local precision = tonumber( frame.args[2] or 0 ); | local precision = tonumber( frame.args[2] or 0 ); | ||
local current_precision = z._precision( value ); | local current_precision = z._precision( value ); | ||
local order = z._order( value ); | |||
-- Due to round-off effects it is neccesary to limit the returned precision under | |||
-- some circumstances because the terminal digits will be inaccurately reported. | |||
if order + precision >= 14 then | |||
local orig_precision = z._precision( value_string ); | |||
if order + orig_precision >= 14 then | |||
precision = 13 - order; | |||
end | |||
end | |||
-- If rounding off, truncate extra digits | -- If rounding off, truncate extra digits | ||
if precision < current_precision then | if precision < current_precision then | ||
| Line 129: | Line 141: | ||
sign = ''; | sign = ''; | ||
end | end | ||
-- Handle cases requiring scientific notation | -- Handle cases requiring scientific notation | ||
if string.find( formatted_num, 'E', 1, true ) ~= nil or math.abs(order) >= 9 then | if string.find( formatted_num, 'E', 1, true ) ~= nil or math.abs(order) >= 9 then | ||
value = value * math.pow( 10, -order ); | value = value * math.pow( 10, -order ); | ||