added code and docs to search for regex within the content of revisions

[wikiq] / wikiq.cpp

diff --git a/wikiq.cpp b/wikiq.cpp
index 712304779da85c02ecca7ef8369d6cb2ecc5ed1f..18259ea836924c24dae060c91177d3829e81e5dd 100644 (file)
--- a/wikiq.cpp
+++ b/wikiq.cpp
@@ -56,9 +56,18 @@ typedef struct {
     char *comment;
     char *text;
     vector<string> last_text_tokens;
     char *comment;
     char *text;
     vector<string> last_text_tokens;

-    vector<pcrecpp::RE> regexes;
+
+    // title regexes

     vector<pcrecpp::RE> title_regexes;
     vector<pcrecpp::RE> title_regexes;

-    vector<string> regex_names;
+
+    // regexes for checking with revisions
+    vector<string> content_regex_names;
+    vector<pcrecpp::RE> content_regexes;
+
+    // regexes for looking within diffs
+    vector<string> diff_regex_names;
+    vector<pcrecpp::RE> diff_regexes;
+

     map<string, string> revision_md5; // used for detecting reversions
 
     // track string size of the elements, to prevent O(N^2) processing in charhndl
     map<string, string> revision_md5; // used for detecting reversions
 
     // track string size of the elements, to prevent O(N^2) processing in charhndl


@@ -264,13 +273,22 @@ write_row(revisionData *data)
         }
     }
 
         }
     }
 

+    // search the content of the revision for a any of the regexes
+    vector<bool> content_regex_matches;
+    if (!data->content_regexes.empty()) {
+        for (vector<pcrecpp::RE>::iterator r = data->content_regexes.begin(); r != data->content_regexes.end(); ++r) {
+            pcrecpp::RE& content_regex = *r;
+            content_regex_matches.push_back(content_regex.PartialMatch(data->text));
+        }
+    }
+

     //vector<string> additions;
     //vector<string> deletions;
     string additions;
     string deletions;
 
     //vector<string> additions;
     //vector<string> deletions;
     string additions;
     string deletions;
 

-    vector<bool> regex_matches_adds;
-    vector<bool> regex_matches_dels;
+    vector<bool> diff_regex_matches_adds;
+    vector<bool> diff_regex_matches_dels;

 
     if (data->last_text_tokens.empty()) {
         additions = data->text;
 
     if (data->last_text_tokens.empty()) {
         additions = data->text;


@@ -298,17 +316,17 @@ write_row(revisionData *data)
     
     if (!additions.empty()) {
         //cout << "ADD: " << additions << endl;
     
     if (!additions.empty()) {
         //cout << "ADD: " << additions << endl;

-        for (vector<pcrecpp::RE>::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) {
-            pcrecpp::RE& regex = *r;
-            regex_matches_adds.push_back(regex.PartialMatch(additions));
+        for (vector<pcrecpp::RE>::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) {
+            pcrecpp::RE& diff_regex = *r;
+            diff_regex_matches_adds.push_back(diff_regex.PartialMatch(additions));

         }
     }
 
     if (!deletions.empty()) {
         //cout << "DEL: " << deletions << endl;
         }
     }
 
     if (!deletions.empty()) {
         //cout << "DEL: " << deletions << endl;

-        for (vector<pcrecpp::RE>::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) {
-            pcrecpp::RE& regex = *r;
-            regex_matches_dels.push_back(regex.PartialMatch(deletions));
+        for (vector<pcrecpp::RE>::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) {
+            pcrecpp::RE& diff_regex = *r;
+            diff_regex_matches_dels.push_back(diff_regex.PartialMatch(deletions));

         }
     }
 
         }
     }
 


@@ -333,9 +351,14 @@ write_row(revisionData *data)
         << (int) additions.size() << "\t"
         << (int) deletions.size();
 
         << (int) additions.size() << "\t"
         << (int) deletions.size();
 

-    for (int n = 0; n < data->regex_names.size(); ++n) {
-        cout << "\t" << ((!regex_matches_adds.empty() && regex_matches_adds.at(n)) ? "TRUE" : "FALSE")
-             << "\t" << ((!regex_matches_dels.empty() && regex_matches_dels.at(n)) ? "TRUE" : "FALSE");
+    for (int n = 0; n < data->content_regex_names.size(); ++n) {
+        cout << "\t" << ((!content_regex_matches.empty()
+                         && content_regex_matches.at(n)) ? "TRUE" : "FALSE");
+    }
+
+    for (int n = 0; n < data->diff_regex_names.size(); ++n) {
+        cout << "\t" << ((!diff_regex_matches_adds.empty() && diff_regex_matches_adds.at(n)) ? "TRUE" : "FALSE")
+             << "\t" << ((!diff_regex_matches_dels.empty() && diff_regex_matches_dels.at(n)) ? "TRUE" : "FALSE");

     }
     cout << endl;
 
     }
     cout << endl;
 


@@ -507,8 +530,10 @@ void print_usage(char* argv[]) {
          << endl
          << "options:" << endl
          << "  -v   verbose mode prints text and comments after each line of tab separated data" << endl
          << endl
          << "options:" << endl
          << "  -v   verbose mode prints text and comments after each line of tab separated data" << endl

-         << "  -n   name of the following regex (e.g. -n name -r \"...\")" << endl
-         << "  -r   regex to check against additions and deletions" << endl
+         << "  -n   name of the following regex for contet (e.g. -n name -r \"...\")" << endl
+         << "  -r   regex to check against content of the revision" << endl
+         << "  -N   name of the following regex for diffs (e.g. -N name -R \"...\")" << endl
+         << "  -R   regex to check against diffs (i.e., additions and deletions)" << endl

          << "  -t   parse revisions only from pages whose titles match regex(es)" << endl
          << endl
          << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl
          << "  -t   parse revisions only from pages whose titles match regex(es)" << endl
          << endl
          << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl


@@ -535,7 +560,8 @@ main(int argc, char *argv[])
     // in "simple" output, we don't print text and comments
     output_type = SIMPLE;
     char c;
     // in "simple" output, we don't print text and comments
     output_type = SIMPLE;
     char c;

-    string regex_name;
+    string diff_regex_name;
+    string content_regex_name;

 
     // the user data struct which is passed to callback functions
     revisionData data;
 
     // the user data struct which is passed to callback functions
     revisionData data;


@@ -550,13 +576,23 @@ main(int argc, char *argv[])
                 output_type = FULL;
                 break;
             case 'n':
                 output_type = FULL;
                 break;
             case 'n':

-                regex_name = optarg;
+                content_regex_name = optarg;

                 break;
             case 'r':
                 break;
             case 'r':

-                data.regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
-                data.regex_names.push_back(regex_name);
-                if (!regex_name.empty()) {
-                    regex_name.clear();
+                data.content_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
+                data.content_regex_names.push_back(content_regex_name);
+                if (!content_regex_name.empty()) {
+                    content_regex_name.clear();
+                }
+                break;
+            case 'N':
+                diff_regex_name = optarg;
+                break;
+            case 'R':
+                data.diff_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
+                data.diff_regex_names.push_back(diff_regex_name);
+                if (!diff_regex_name.empty()) {
+                    diff_regex_name.clear();

                 }
                 break;
             case 'h':
                 }
                 break;
             case 'h':


@@ -613,17 +649,29 @@ main(int argc, char *argv[])
         << "deletions_size";
 
     int n = 0;
         << "deletions_size";
 
     int n = 0;

-    if (!data.regexes.empty()) {
-        for (vector<pcrecpp::RE>::iterator r = data.regexes.begin(); r != data.regexes.end(); ++r, ++n) {
-            if (data.regex_names.at(n).empty()) {
+    if (!data.content_regexes.empty()) {
+        for (vector<pcrecpp::RE>::iterator r = data.content_regexes.begin();
+            r != data.content_regexes.end(); ++r, ++n) {
+            if (data.content_regex_names.at(n).empty()) {
+               cout << "\t" << "regex" << n;
+            } else {
+               cout << "\t" << data.content_regex_names.at(n);
+            }
+        }
+    }
+
+    if (!data.diff_regexes.empty()) {
+        for (vector<pcrecpp::RE>::iterator r = data.diff_regexes.begin(); r != data.diff_regexes.end(); ++r, ++n) {
+            if (data.diff_regex_names.at(n).empty()) {

                 cout << "\t" << "regex_" << n << "_add"
                      << "\t" << "regex_" << n << "_del";
             } else {
                 cout << "\t" << "regex_" << n << "_add"
                      << "\t" << "regex_" << n << "_del";
             } else {

-                cout << "\t" << data.regex_names.at(n) << "_add"
-                     << "\t" << data.regex_names.at(n) << "_del";
+                cout << "\t" << data.diff_regex_names.at(n) << "_add"
+                     << "\t" << data.diff_regex_names.at(n) << "_del";

             }
         }
     }
             }
         }
     }

+

     cout << endl;
     
     // shovel data into the parser
     cout << endl;
     
     // shovel data into the parser