mhnsw: make the search less greedy

introduced a generosity factor that makes the search less greedy.
it dramatically improves the recall by making the search a bit slower
(for the same recall one can use half the M and smaller ef).

had to add Queue::safe_push() method that removes one of the
furthest elements (not necessarily the furthest) in the queue
to keep it from overflowing.

sql/sql_queue.h

@@ -44,6 +44,11 @@ class Queue
   Element *top() const { return (Element*)queue_top(&m_queue); }
 
   void push(const Element *element) { queue_insert(&m_queue, (uchar*)element); }
+  void safe_push(const Element *element)
+  {
+    if (is_full()) m_queue.elements--; // remove one of the furthest elements
+    queue_insert(&m_queue, (uchar*)element);
+  }
   Element *pop() { return (Element *)queue_remove_top(&m_queue); }
   void clear() { queue_remove_all(&m_queue); }
   uint propagate_top() { return queue_replace_top(&m_queue); }

sql/vector_mhnsw.cc

@@ -30,6 +30,7 @@ ulonglong mhnsw_cache_size;
 
 // Algorithm parameters
 static constexpr double alpha = 1.1;
+static constexpr double generosity = 1.2;
 static constexpr double stiffness = 0.002;
 static constexpr uint ef_construction_max_factor= 16;
 static constexpr uint clo_nei_threshold= 10000;
@@ -981,16 +982,18 @@ static int search_layer(MHNSW_Context *ctx, TABLE *graph, const FVector *target,
           if (skip_deleted && v->node->deleted)
             continue;
           best.push(v);
-          furthest_best= best.top()->distance_to_target;
+          furthest_best= best.top()->distance_to_target * generosity;
         }
         else if (v->distance_to_target < furthest_best)
         {
-          candidates.push(v);
+          candidates.safe_push(v);
           if (skip_deleted && v->node->deleted)
             continue;
-          if (best.replace_top(v) <= expand_size)
+          if ((generosity > 1 &&
+               v->distance_to_target >= best.top()->distance_to_target)
+              || best.replace_top(v) <= expand_size)
             v->expand= true;
-          furthest_best= best.top()->distance_to_target;
+          furthest_best= best.top()->distance_to_target * generosity;
         }
       }
     }