Bug 705910 - Indexing and searching cannot treat non ASCII identifiers

src/index.cpp

@@ -138,31 +138,6 @@ class MemberIndexList : public QList<MemberDef>
     uint m_letter;
 };
 
-/** @brief maps a unicode character code to a list of T::ElementType's
- */
-template<class T>
-class LetterToIndexMap : public SIntDict<T>
-{
-  public:
-    LetterToIndexMap() { SIntDict<T>::setAutoDelete(TRUE); }
-    int compareItems(QCollection::Item item1, QCollection::Item item2)
-    {
-      T *l1=(T *)item1;
-      T *l2=(T *)item2;
-      return (int)l1->letter()-(int)l2->letter();
-    }
-    void append(uint letter,typename T::ElementType *elem)
-    {
-      T *l = SIntDict<T>::find((int)letter);
-      if (l==0)
-      {
-        l = new T(letter);
-        SIntDict<T>::inSort((int)letter,l);
-      }
-      l->append(elem);
-    }
-};
-
 static LetterToIndexMap<MemberIndexList> g_memberIndexLetterUsed[CMHL_Total];
 static LetterToIndexMap<MemberIndexList> g_fileIndexLetterUsed[FMHL_Total];
 static LetterToIndexMap<MemberIndexList> g_namespaceIndexLetterUsed[NMHL_Total];
@@ -1783,7 +1758,7 @@ class PrefixIgnoreClassList : public ClassList
 class AlphaIndexTableCell
 {
   public:
-    AlphaIndexTableCell(int row,int col,uchar letter,ClassDef *cd) : 
+    AlphaIndexTableCell(int row,int col,uint letter,ClassDef *cd) : 
       m_letter(letter), m_class(cd), m_row(row), m_col(col) 
     { //printf("AlphaIndexTableCell(%d,%d,%c,%s)\n",row,col,letter!=0 ? letter: '-',
       //       cd!=(ClassDef*)0x8 ? cd->name().data() : "<null>"); 
@@ -1914,7 +1889,7 @@ static void writeAlphabeticalClassList(OutputList &ol)
     if (cd->isLinkableInProject() && cd->templateMaster()==0)
     {
       int index = getPrefixIndex(cd->className());
-      startLetter=toupper(cd->className().at(index))&0xFF;
+      startLetter=getUtf8Code(cd->className(),index);
       // Do some sorting again, since the classes are sorted by name with 
       // prefix, which should be ignored really.
       if (cd->getLanguage()==SrcLangExt_VHDL)
@@ -1954,7 +1929,7 @@ static void writeAlphabeticalClassList(OutputList &ol)
   {
     uint l = cl->letter();
     // add special header cell
-    tableRows->append(new AlphaIndexTableCell(row,col,(uchar)l,(ClassDef*)0x8));
+    tableRows->append(new AlphaIndexTableCell(row,col,l,(ClassDef*)0x8));
     row++;
     tableRows->append(new AlphaIndexTableCell(row,col,0,(ClassDef*)0x8));
     row++;
@@ -2017,7 +1992,7 @@ static void writeAlphabeticalClassList(OutputList &ol)
               ol.writeString("<table border=\"0\" cellspacing=\"0\" cellpadding=\"0\">"
                   "<tr>"
                   "<td><div class=\"ah\">&#160;&#160;"); 
-              ol.writeString(s);
+              ol.writeString(QString(QChar(cell->letter())).utf8());
               ol.writeString(         "&#160;&#160;</div>"
                   "</td>"
                   "</tr>"

src/search.js

@@ -5,7 +5,7 @@ function convertToId(search)
   {
     var c = search.charAt(i);
     var cn = c.charCodeAt(0);
-    if (c.match(/[a-z0-9]/))
+    if (c.match(/[a-z0-9\u0080-\uFFFF]/))
     {
       result+=c;
     }
@@ -310,22 +310,20 @@ function SearchBox(name, resultsPath, inFrame, label)
     var searchValue = this.DOMSearchField().value.replace(/^ +/, "");
 
     var code = searchValue.toLowerCase().charCodeAt(0);
-    var hexCode;
-    if (code<16) 
+    var idxChar = searchValue.substr(0, 1).toLowerCase();
+    if ( 0xD800 <= code && code <= 0xDBFF && searchValue > 1) // surrogate pair
     {
-      hexCode="0"+code.toString(16);
-    }
-    else 
-    {
-      hexCode=code.toString(16);
+      idxChar = searchValue.substr(0, 2);
     }
 
     var resultsPage;
     var resultsPageWithSearch;
     var hasResultsPage;
 
-    if (indexSectionsWithContent[this.searchIndex].charAt(code) == '1')
+    var idx = indexSectionsWithContent[this.searchIndex].indexOf(idxChar);
+    if (idx!=-1)
     {
+       var hexCode=idx.toString(16);
        resultsPage = this.resultsPath + '/' + indexSectionNames[this.searchIndex] + '_' + hexCode + '.html';
        resultsPageWithSearch = resultsPage+'?'+escape(searchValue);
        hasResultsPage = true;

src/search_functions.php

@@ -358,7 +358,7 @@ function main()
   $sorted = run_query($query);
   // Now output the HTML stuff...
   // End the HTML form
-  end_form(preg_replace("/[^a-zA-Z0-9\-\_\.]/i", " ", $query ));
+  end_form(preg_replace("/[^a-zA-Z0-9\-\_\.\x80-\xFF]/i", " ", $query ));
   // report results to the user
   report_results($sorted);
   end_page();

src/search_functions_php.h

@@ -358,7 +358,7 @@
 "  $sorted = run_query($query);\n"
 "  // Now output the HTML stuff...\n"
 "  // End the HTML form\n"
-"  end_form(preg_replace(\"/[^a-zA-Z0-9\\-\\_\\.]/i\", \" \", $query ));\n"
+"  end_form(preg_replace(\"/[^a-zA-Z0-9\\-\\_\\.\\x80-\\xFF]/i\", \" \", $query ));\n"
 "  // report results to the user\n"
 "  report_results($sorted);\n"
 "  end_page();\n"

src/search_js.h

@@ -5,7 +5,7 @@
 "  {\n"
 "    var c = search.charAt(i);\n"
 "    var cn = c.charCodeAt(0);\n"
-"    if (c.match(/[a-z0-9]/))\n"
+"    if (c.match(/[a-z0-9\\u0080-\\uFFFF]/))\n"
 "    {\n"
 "      result+=c;\n"
 "    }\n"
@@ -310,22 +310,20 @@
 "    var searchValue = this.DOMSearchField().value.replace(/^ +/, \"\");\n"
 "\n"
 "    var code = searchValue.toLowerCase().charCodeAt(0);\n"
-"    var hexCode;\n"
-"    if (code<16) \n"
+"    var idxChar = searchValue.substr(0, 1).toLowerCase();\n"
+"    if ( 0xD800 <= code && code <= 0xDBFF && searchValue > 1) // surrogate pair\n"
 "    {\n"
-"      hexCode=\"0\"+code.toString(16);\n"
-"    }\n"
-"    else \n"
-"    {\n"
-"      hexCode=code.toString(16);\n"
+"      idxChar = searchValue.substr(0, 2);\n"
 "    }\n"
 "\n"
 "    var resultsPage;\n"
 "    var resultsPageWithSearch;\n"
 "    var hasResultsPage;\n"
 "\n"
-"    if (indexSectionsWithContent[this.searchIndex].charAt(code) == '1')\n"
+"    var idx = indexSectionsWithContent[this.searchIndex].indexOf(idxChar);\n"
+"    if (idx!=-1)\n"
 "    {\n"
+"       var hexCode=idx.toString(16);\n"
 "       resultsPage = this.resultsPath + '/' + indexSectionNames[this.searchIndex] + '_' + hexCode + '.html';\n"
 "       resultsPageWithSearch = resultsPage+'?'+escape(searchValue);\n"
 "       hasResultsPage = true;\n"

src/util.h

@@ -25,6 +25,7 @@
 #include <qlist.h>
 #include <ctype.h>
 #include "types.h"
+#include "sortdict.h"
 
 //--------------------------------------------------------------------
 
@@ -87,6 +88,33 @@ class TextGeneratorOLImpl : public TextGeneratorIntf
 
 //--------------------------------------------------------------------
 
+/** @brief maps a unicode character code to a list of T::ElementType's
+ */
+template<class T>
+class LetterToIndexMap : public SIntDict<T>
+{
+  public:
+    LetterToIndexMap() { SIntDict<T>::setAutoDelete(TRUE); }
+    int compareItems(QCollection::Item item1, QCollection::Item item2)
+    {
+      T *l1=(T *)item1;
+      T *l2=(T *)item2;
+      return (int)l1->letter()-(int)l2->letter();
+    }
+    void append(uint letter,typename T::ElementType *elem)
+    {
+      T *l = SIntDict<T>::find((int)letter);
+      if (l==0)
+      {
+        l = new T(letter);
+        SIntDict<T>::inSort((int)letter,l);
+      }
+      l->append(elem);
+    }
+};
+
+//--------------------------------------------------------------------
+
 QCString langToString(SrcLangExt lang);
 QCString getLanguageSpecificSeparator(SrcLangExt lang,bool classScope=FALSE);