Merge branch 'patch-3' of github.com:nemobis/wikiq

[wikiq] / wikiq.cpp

diff --git a/wikiq.cpp b/wikiq.cpp
index dfa7251d5a9e0ea943be334480cc8539fb19094a..ad9ac48fcb99825dddf6d80a44f23ebe3c88a5b1 100644 (file)
--- a/wikiq.cpp
+++ b/wikiq.cpp
@@ -11,7 +11,6 @@
 #include <stdlib.h>
 #include "expat.h"
 #include <getopt.h>
-#include "disorder.h"
 #include "md5.h"
 #include "dtl/dtl.hpp"
 #include <vector>
@@ -56,9 +55,18 @@ typedef struct {
     char *comment;
     char *text;
     vector<string> last_text_tokens;
+
+    // title regexes
     vector<pcrecpp::RE> title_regexes;
+
+    // regexes for checking with revisions
+    vector<string> content_regex_names;
+    vector<pcrecpp::RE> content_regexes;
+
+    // regexes for looking within diffs
     vector<string> diff_regex_names;
     vector<pcrecpp::RE> diff_regexes;
+
     map<string, string> revision_md5; // used for detecting reversions
 
     // track string size of the elements, to prevent O(N^2) processing in charhndl
@@ -264,6 +272,15 @@ write_row(revisionData *data)
         }
     }
 
+    // search the content of the revision for a any of the regexes
+    vector<bool> content_regex_matches;
+    if (!data->content_regexes.empty()) {
+        for (vector<pcrecpp::RE>::iterator r = data->content_regexes.begin(); r != data->content_regexes.end(); ++r) {
+            pcrecpp::RE& content_regex = *r;
+            content_regex_matches.push_back(content_regex.PartialMatch(data->text));
+        }
+    }
+
     //vector<string> additions;
     //vector<string> deletions;
     string additions;
@@ -327,12 +344,16 @@ write_row(revisionData *data)
         << data->editorid << "\t"
         << ((data->minor) ? "TRUE" : "FALSE") << "\t"
         << (unsigned int) data->text_size << "\t"
-        << shannon_H(data->text, data->text_size) << "\t"
         << md5_hex_output << "\t"
         << reverted_to << "\t"
         << (int) additions.size() << "\t"
         << (int) deletions.size();
 
+    for (int n = 0; n < data->content_regex_names.size(); ++n) {
+        cout << "\t" << ((!content_regex_matches.empty()
+                         && content_regex_matches.at(n)) ? "TRUE" : "FALSE");
+    }
+
     for (int n = 0; n < data->diff_regex_names.size(); ++n) {
         cout << "\t" << ((!diff_regex_matches_adds.empty() && diff_regex_matches_adds.at(n)) ? "TRUE" : "FALSE")
              << "\t" << ((!diff_regex_matches_dels.empty() && diff_regex_matches_dels.at(n)) ? "TRUE" : "FALSE");
@@ -507,22 +528,25 @@ void print_usage(char* argv[]) {
          << endl
          << "options:" << endl
          << "  -v   verbose mode prints text and comments after each line of tab separated data" << endl
-         << "  -n   name of the following regex (e.g. -n name -r \"...\")" << endl
-         << "  -r   regex to check against additions and deletions" << endl
+         << "  -n   name of the following regex for content (e.g. -n name -r \"...\")" << endl
+         << "  -r   regex to check against content of the revision" << endl
+         << "  -N   name of the following regex for diffs (e.g. -N name -R \"...\")" << endl
+         << "  -R   regex to check against diffs (i.e., additions and deletions)" << endl
          << "  -t   parse revisions only from pages whose titles match regex(es)" << endl
          << endl
          << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl
          << "a tab-separated stream of revisions on standard out:" << endl
          << endl
          << "title, articleid, revid, timestamp, anon, editor, editorid, minor," << endl
-         << "text_length, text_entropy, text_md5, reversion, additions_size, deletions_size" << endl
+         << "text_length, text_md5, reversion, additions_size, deletions_size" << endl
          << ".... and additional fields for each regex executed against add/delete diffs" << endl
          << endl
          << "Boolean fields are TRUE/FALSE except in the case of reversion, which is blank" << endl
          << "unless the article is a revert to a previous revision, in which case, it" << endl
          << "contains the revision ID of the revision which was reverted to." << endl
          << endl
-         << "author: Erik Garrison <erik@hypervolu.me>" << endl;
+         << "authors: Erik Garrison <erik@hypervolu.me>" << endl
+         << "         Benjamin Mako Hill <mako@atdot.cc>" << endl;
 }
 
 
@@ -536,6 +560,7 @@ main(int argc, char *argv[])
     output_type = SIMPLE;
     char c;
     string diff_regex_name;
+    string content_regex_name;
 
     // the user data struct which is passed to callback functions
     revisionData data;
@@ -550,9 +575,19 @@ main(int argc, char *argv[])
                 output_type = FULL;
                 break;
             case 'n':
-                diff_regex_name = optarg;
+                content_regex_name = optarg;
                 break;
             case 'r':
+                data.content_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
+                data.content_regex_names.push_back(content_regex_name);
+                if (!content_regex_name.empty()) {
+                    content_regex_name.clear();
+                }
+                break;
+            case 'N':
+                diff_regex_name = optarg;
+                break;
+            case 'R':
                 data.diff_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
                 data.diff_regex_names.push_back(diff_regex_name);
                 if (!diff_regex_name.empty()) {
@@ -599,20 +634,30 @@ main(int argc, char *argv[])
     cout << "title" << "\t"
         << "articleid" << "\t"
         << "revid" << "\t"
-        << "date" << " "
+        << "date" << "_"
         << "time" << "\t"
         << "anon" << "\t"
         << "editor" << "\t"
         << "editor_id" << "\t"
         << "minor" << "\t"
         << "text_size" << "\t"
-        << "text_entropy" << "\t"
         << "text_md5" << "\t"
         << "reversion" << "\t"
         << "additions_size" << "\t"
         << "deletions_size";
 
     int n = 0;
+    if (!data.content_regexes.empty()) {
+        for (vector<pcrecpp::RE>::iterator r = data.content_regexes.begin();
+            r != data.content_regexes.end(); ++r, ++n) {
+            if (data.content_regex_names.at(n).empty()) {
+               cout << "\t" << "regex" << n;
+            } else {
+               cout << "\t" << data.content_regex_names.at(n);
+            }
+        }
+    }
+
     if (!data.diff_regexes.empty()) {
         for (vector<pcrecpp::RE>::iterator r = data.diff_regexes.begin(); r != data.diff_regexes.end(); ++r, ++n) {
             if (data.diff_regex_names.at(n).empty()) {
@@ -624,6 +669,7 @@ main(int argc, char *argv[])
             }
         }
     }
+
     cout << endl;
     
     // shovel data into the parser