Jump to content

Module:Math: Difference between revisions

403 bytes added ,  11 years ago
problem with lost precision
m>Dragons flight
m (precision truncation issue)
m>Dragons flight
(problem with lost precision)
Line 109: Line 109:
     local lang = mw.getContentLanguage();
     local lang = mw.getContentLanguage();
      
      
     local value = tonumber( frame.args[1] or 0 );
     local value_string = ( frame.args[1] or '0' )
    local value = tonumber( value_string );
     local precision = tonumber( frame.args[2] or 0 );
     local precision = tonumber( frame.args[2] or 0 );
     local current_precision = z._precision( value );
     local current_precision = z._precision( value );
    local order = z._order( value );
      
      
    -- Due to round-off effects it is neccesary to limit the returned precision under
    -- some circumstances because the terminal digits will be inaccurately reported.
    if order + precision >= 14 then
        local orig_precision = z._precision( value_string );
        if order + orig_precision >= 14 then
            precision = 13 - order;
        end
    end
     -- If rounding off, truncate extra digits
     -- If rounding off, truncate extra digits
     if precision < current_precision then
     if precision < current_precision then
Line 129: Line 141:
         sign = '';
         sign = '';
     end     
     end     
   
       
    local order;
   
     -- Handle cases requiring scientific notation
     -- Handle cases requiring scientific notation
    order = z._order( value );
     if string.find( formatted_num, 'E', 1, true ) ~= nil or math.abs(order) >= 9 then
     if string.find( formatted_num, 'E', 1, true ) ~= nil or math.abs(order) >= 9 then
         value = value * math.pow( 10, -order );
         value = value * math.pow( 10, -order );
Anonymous user
Cookies help us deliver our services. By using our services, you agree to our use of cookies.