Fix bug #10802 Index is not used if table using BDB engine on HP-UX

HP-UX compiler was generating code which makes wrong conversion from double to
ulonglong which results in not using index in some cases.

Changed type cast operators odrer with which generated code makes proper 
conversion.

mysql-test/include/varchar.inc

@@ -229,6 +229,7 @@ drop table t1;
 
 #
 # Bug #9489: problem with hash indexes
+# Bug #10802: Index is not used if table using BDB engine on HP-UX
 #
 
 create table t1(a int, b varchar(12), key ba(b, a));

sql/ha_berkeley.cc

@@ -2150,7 +2150,7 @@ ha_rows ha_berkeley::records_in_range(uint keynr, key_range *start_key,
     end_pos=end_range.less+end_range.equal;
   rows=(end_pos-start_pos)*records;
   DBUG_PRINT("exit",("rows: %g",rows));
-  DBUG_RETURN(rows <= 1.0 ? (ha_rows) 1 : (ha_rows) rows);
+  DBUG_RETURN((ha_rows)(rows <= 1.0 ? 1 : rows));
 }