- PluggableIndexes interface: + indexSize()

- browser for stored values for all UnIndex derived indexes

doc/CHANGES.txt

@@ -30,6 +30,13 @@ Zope Changes
 
     Features added
 
+      - PluginIndexes: the ZCatalog's "Indexes" tab now show the number of
+        distinct values indexed by each index instead of a mixture of indexed
+        objects versus number of distinct values. Indexes derived from UnIndex 
+        show both values within their own ZMI screen. In addition most indexes 
+        have now a "Browse" tab to browse through the list of indexed 
+        values and their occurrences.
+
       - FTPServer: a RNFR (rename from) request is now being responded
         with a 550 error code if the source file does not exist
 

lib/python/Products/PluginIndexes/DateIndex/DateIndex.py

@@ -88,17 +88,22 @@ class DateIndex(UnIndex, PropertyManager):
                   'mode':'w'},)
 
     manage = manage_main = DTMLFile( 'dtml/manageDateIndex', globals() )
+    manage_browse = DTMLFile('../dtml/browseIndex', globals())
+
     manage_main._setName( 'manage_main' )
     manage_options = ( { 'label' : 'Settings'
                        , 'action' : 'manage_main'
                        },
+                       {'label': 'Browse',
+                        'action': 'manage_browse',
+                       },
                      ) + PropertyManager.manage_options
 
     def clear( self ):
         """ Complete reset """
         self._index = IOBTree()
         self._unindex = OIBTree()
-
+        self._length.set(0)
 
     def index_object( self, documentId, obj, threshold=None ):
         """index an object, normalizing the indexed value to an integer

lib/python/Products/PluginIndexes/DateIndex/dtml/manageDateIndex.dtml

@@ -2,7 +2,9 @@
 <dtml-var manage_tabs>
 
 <p class="form-help">
-Nothing to manage at this time.
+Objects indexed: <dtml-var numObjects>
+<br>
+Distinct values: <dtml-var indexSize>
 </p>
 
 

lib/python/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py

@@ -25,6 +25,7 @@ from OFS.SimpleItem import SimpleItem
 
 from BTrees.IOBTree import IOBTree
 from BTrees.IIBTree import IISet, IITreeSet, union, intersection, multiunion
+import BTrees.Length
 
 from Globals import package_home, DTMLFile, InitializeClass
 from AccessControl import ClassSecurityInfo
@@ -136,6 +137,7 @@ class DateRangeIndex(UnIndex):
         self._since         = IOBTree()
         self._until         = IOBTree()
         self._unindex       = IOBTree() # 'datum' will be a tuple of date ints
+        self._length        = BTrees.Length.Length()
 
     #
     #   PluggableIndexInterface implementation (XXX inherit assertions?)
@@ -310,10 +312,13 @@ class DateRangeIndex(UnIndex):
     #
     security.declareProtected( VIEW_PERMISSION , 'numObjects' )
     def numObjects( self ):
-        """
-        """
+        """ """
         return len( self._unindex )
 
+    def indexSize(self):
+        """ """
+        return len(self)
+
     #
     #   Helper functions.
     #

lib/python/Products/PluginIndexes/DateRangeIndex/dtml/manageDateRangeIndex.dtml

@@ -6,6 +6,12 @@ You can update this DateRangeIndex by editing the following field and clicking
 <emUpdate</em>.
 </p>
 
+<p>
+Objects indexed: <dtml-var numObjects>
+<br>
+Distinct values: <dtml-var indexSize>
+</p>
+
 <form action="&dtml-URL1;/manage_edit" method="POST">
 <table cellpadding="2" cellspacing="0" border="0">
 <tr>

lib/python/Products/PluginIndexes/FieldIndex/FieldIndex.py

@@ -32,6 +32,9 @@ class FieldIndex(UnIndex):
         {'label': 'Settings',
          'action': 'manage_main',
          'help': ('FieldIndex','FieldIndex_Settings.stx')},
+        {'label': 'Browse',
+         'action': 'manage_browse',
+         'help': ('FieldIndex','FieldIndex_Settings.stx')},
     )
 
     query_options = ["query","range"]
@@ -39,6 +42,7 @@ class FieldIndex(UnIndex):
     index_html = DTMLFile('dtml/index', globals())
 
     manage_workspace = DTMLFile('dtml/manageFieldIndex', globals())
+    manage_browse = DTMLFile('../dtml/browseIndex', globals())
 
 
 manage_addFieldIndexForm = DTMLFile('dtml/addFieldIndex', globals())

lib/python/Products/PluginIndexes/FieldIndex/dtml/manageFieldIndex.dtml

@@ -2,7 +2,9 @@
 <dtml-var manage_tabs>
 
 <p class="form-help">
-Nothing to manage at this time.
+Objects indexed: <dtml-var numObjects>
+<br>
+Distinct values: <dtml-var indexSize>
 </p>
 
 

lib/python/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py

@@ -116,7 +116,7 @@ class TestCase( unittest.TestCase ):
 
         assert len( self._index ) == len( values )-1 #'abce' is duplicate
         assert len( self._index.referencedObjects() ) == len( values )
-        self.assertEqual(self._index.numObjects(), len( values )-1)
+        self.assertEqual(self._index.indexSize(), len( values )-1)
 
         assert self._index.getEntryForObject( 1234 ) is None
         assert ( self._index.getEntryForObject( 1234, self._marker )

lib/python/Products/PluginIndexes/KeywordIndex/KeywordIndex.py

@@ -34,6 +34,9 @@ class KeywordIndex(UnIndex):
         {'label': 'Settings',
          'action': 'manage_main',
          'help': ('KeywordIndex','KeywordIndex_Settings.stx')},
+        {'label': 'Browse',
+         'action': 'manage_browse',
+         'help': ('FieldIndex','FieldIndex_Settings.stx')},
     )
 
     query_options = ("query","operator", "range")
@@ -128,6 +131,7 @@ class KeywordIndex(UnIndex):
 
     index_html = DTMLFile('dtml/index', globals())
     manage_workspace = DTMLFile('dtml/manageKeywordIndex', globals())
+    manage_browse = DTMLFile('../dtml/browseIndex', globals())
 
 
 

lib/python/Products/PluginIndexes/KeywordIndex/dtml/manageKeywordIndex.dtml

@@ -2,7 +2,9 @@
 <dtml-var manage_tabs>
 
 <p class="form-help">
-Nothing to manage at this time.
+Objects indexed: <dtml-var numObjects>
+<br>
+Distinct values: <dtml-var indexSize>
 </p>
 
 

lib/python/Products/PluginIndexes/KeywordIndex/tests/testKeywordIndex.py

@@ -142,7 +142,7 @@ class TestKeywordIndex( unittest.TestCase ):
         assert ( self._index.getEntryForObject( 1234, self._marker )
                   is self._marker )
         self._index.unindex_object( 1234 ) # nothrow
-        self.assertEqual(self._index.numObjects(), len( values )-1)
+        self.assertEqual(self._index.indexSize(), len( values )-1)
 
         for k, v in values:
             entry = self._index.getEntryForObject( k )

lib/python/Products/PluginIndexes/PathIndex/PathIndex.py

@@ -113,7 +113,6 @@ class PathIndex(Persistent, SimpleItem):
         comps = filter(None, path.split('/'))
        
         if not self._unindex.has_key(docid):
-            self._migrate_length()
             self._length.change(1)
 
         for i in range(len(comps)):
@@ -146,7 +145,6 @@ class PathIndex(Persistent, SimpleItem):
                 LOG.error('Attempt to unindex document with id %s failed'
                           % docid)
 
-        self._migrate_length()
         self._length.change(-1)
         del self._unindex[docid]
 
@@ -200,14 +198,15 @@ class PathIndex(Persistent, SimpleItem):
             return results
 
     def numObjects(self):
+        """ return the number distinct values """
+        return len(self._unindex)
+
+    def indexSize(self):
         """ return the number of indexed objects"""
-        self._migrate_length()
-        return self._length()
+        return len(self)
 
-    def _migrate_length(self):
-        """ migrate index to use new _length attribute """
-        if not hasattr(self, '_length'):
-            self._length = Length(len(self._unindex))
+    def __len__(self):
+        return self._length()
 
     def _apply_index(self, request, cid=''):
         """ hook for (Z)Catalog

lib/python/Products/PluginIndexes/PathIndex/dtml/managePathIndex.dtml

@@ -2,7 +2,9 @@
 <dtml-var manage_tabs>
 
 <p class="form-help">
-Nothing to manage at this time.
+Objects indexed: <dtml-var numObjects>
+<br>
+Distinct values: <dtml-var indexSize>
 </p>
 
 

lib/python/Products/PluginIndexes/common/PluggableIndex.py

@@ -67,6 +67,9 @@ class PluggableIndexInterface(Interface.Base):
     
     def numObjects():
         """Return the number of indexed objects"""
+
+    def indexSize():
+        """Return the size of the index in terms of distinct values"""
     
     def clear():
         """Empty the index"""

lib/python/Products/PluginIndexes/dtml/browseIndex.dtml

+<dtml-var manage_page_header>
+<dtml-var manage_tabs>
+
+<dtml-call "REQUEST.RESPONSE.setHeader('Content-Type', 'text/html; charset=UTF-8')" >
+
+<p class="form-text">
+The index "&dtml-getId;" contains <dtml-var items fmt=collection-length thousands_commas> distinct values
+</p>
+
+<dtml-let size="20"> <!-- batch size -->
+
+  <div class="form-text">
+    <dtml-in items previous size=size start=query_start >
+        <a href="&dtml-URL;?query_start=&dtml-previous-sequence-start-number;">
+        [Previous <dtml-var previous-sequence-size> entries]
+      </a>
+    </dtml-in>
+    <dtml-in items next size=size start=query_start >
+      <a href="&dtml-URL;?query_start=&dtml-next-sequence-start-number;">
+        [Next <dtml-var next-sequence-size> entries]
+      </a>
+    </dtml-in>
+  </div>
+
+  <table border="1" align="center" width="100%" class="form-help">
+    <dtml-in items start=query_start size=size>
+      <tr>
+        <td>
+          <dtml-if "meta_type in ('DateIndex',)">
+            <dtml-var "DateTime(_['sequence-key'])">
+          <dtml-else>
+            &dtml-sequence-key; 
+          </dtml-if>
+        </td>
+        <td>   
+          <ul>
+            <dtml-let v="_['sequence-item']">
+              <dtml-if "isinstance(v, int)">
+                <li><a href="<dtml-var "getpath(v)">"<dtml-var "getpath(v)"></a></li>    
+              <dtml-else>
+                <dtml-in "v.keys()">
+                  <li> <a href="<dtml-var "getpath(_['sequence-item'])">"><dtml-var "getpath(_['sequence-item'])"></a></li>
+                </dtml-in>
+              </dtml-if>
+            </dtml-let>
+          </ul>
+        </td>
+      </tr>
+    </dtml-in>
+  </table>
+</dtml-let>
+
+<dtml-var manage_page_footer>

lib/python/Products/ZCTextIndex/ZCTextIndex.py

@@ -217,6 +217,10 @@ class ZCTextIndex(Persistent, Acquisition.Implicit, SimpleItem):
         """Return number of unique words in the index"""
         return self.index.length()
 
+    def indexSize(self):
+        """Return the number of indexes objects """
+        return self.index.document_count()
+
     def clear(self):
         """reinitialize the index (but not the lexicon)"""
         try:

lib/python/Products/ZCatalog/ZCatalog.py

@@ -908,7 +908,8 @@ class ZCatalog(Folder, Persistent, Implicit):
            __len__ changed in Zope 2.8. Pre-Zope 2.7 installation used to implement
            __len__ as persistent attribute of the index instance which is totally
            incompatible with the new extension class implementation based on new-style
-           classes.
+           classes. CMF indexes on date fields will be converted to DateIndex and 
+           DateRangeIndex.
         """
 
         LOG.info('Start migration of indexes for %s' % self.absolute_url(1))
@@ -925,6 +926,10 @@ class ZCatalog(Folder, Persistent, Implicit):
                 idx_type = idx.meta_type
                 idx_id = idx.getId()
                 LOG.info('processing index %s' % idx_id)
+                if idx_type == 'FieldIndex' and idx_id in ('start', 'modified', 'end', 'created'):
+                    idx_type = 'DateIndex'
+                if idx_type == 'FieldIndex' and idx_id in ('effective', 'expires'):
+                    idx_type = 'DateRangeIndex'
                 indexed_attrs = getattr(idx, 'indexed_attrs', None)
                 self.delIndex(idx.getId())
                 self.addIndex(idx_id, idx_type)

lib/python/Products/ZCatalog/dtml/catalogIndexes.dtml

@@ -104,13 +104,13 @@ function toggleSelect() {
   </td>
 
   <td width="20%" align="left"><div class="list-item"><a 
-   href="./manage_catalogIndexes?skey=numObjects<dtml-if 
-   "rkey == ''">&rkey=numObjects</dtml-if
+   href="./manage_catalogIndexes?skey=indexSize<dtml-if 
+   "rkey == ''">&rkey=indexSize</dtml-if
    >"
-   onMouseOver="window.status='Sort objects by number of indexed objects'; return true"
+   onMouseOver="window.status='Sort objects by number of distinct values indexed'; return true"
    onMouseOut="window.status=''; return true"><dtml-if 
-   "skey == 'numObjects' or rkey == 'numObjects'"
-   ><strong># objects</strong><dtml-else># objects</dtml-if></a></div>
+   "skey == 'indexSize' or rkey == 'indexSize'"
+   ><strong># distinct values</strong><dtml-else># distinct values</dtml-if></a></div>
   </td>
 
   <td width="20%" align="left"><div class="list-item"><a 
@@ -169,7 +169,7 @@ function toggleSelect() {
 
     <td>
       <div class="list-item">
-        <dtml-var numObjects missing="n/a">
+        <dtml-var indexSize missing="n/a">
       </div>
     </td>
 
@@ -183,6 +183,7 @@ function toggleSelect() {
 
 </tr>
 </dtml-in>
+
 </table>
 
 <table cellspacing="0" cellpadding="2" border="0">