X-Git-Url: https://projects.mako.cc/source/wikiq/blobdiff_plain/59363fd9d5c67c2ec370d436c329508c4b7a6f54..e5ef5d83494fb4a41e443b8c2586f0fd4cdd8e5b:/wikiq.cpp?ds=sidebyside diff --git a/wikiq.cpp b/wikiq.cpp index da5e7b7..1dce957 100644 --- a/wikiq.cpp +++ b/wikiq.cpp @@ -11,7 +11,6 @@ #include #include "expat.h" #include -#include "disorder.h" #include "md5.h" #include "dtl/dtl.hpp" #include @@ -56,8 +55,18 @@ typedef struct { char *comment; char *text; vector last_text_tokens; - vector regexes; - vector regex_names; + + // title regexes + vector title_regexes; + + // regexes for checking with revisions + vector content_regex_names; + vector content_regexes; + + // regexes for looking within diffs + vector diff_regex_names; + vector diff_regexes; + map revision_md5; // used for detecting reversions // track string size of the elements, to prevent O(N^2) processing in charhndl @@ -246,13 +255,39 @@ write_row(revisionData *data) ++pos; } + // look to see if (a) we've passed in a list of /any/ title_regexes + // and (b) if all of the title_regex_matches match + // if (a) is true and (b) is not, we return + bool any_title_regex_match = false; + if (!data->title_regexes.empty()) { + for (vector::iterator r = data->title_regexes.begin(); r != data->title_regexes.end(); ++r) { + pcrecpp::RE& title_regex = *r; + if (title_regex.PartialMatch(data->title)) { + any_title_regex_match = true; + break; + } + } + if (!any_title_regex_match) { + return; + } + } + + // search the content of the revision for a any of the regexes + vector content_regex_matches; + if (!data->content_regexes.empty()) { + for (vector::iterator r = data->content_regexes.begin(); r != data->content_regexes.end(); ++r) { + pcrecpp::RE& content_regex = *r; + content_regex_matches.push_back(content_regex.PartialMatch(data->text)); + } + } + //vector additions; //vector deletions; string additions; string deletions; - vector regex_matches_adds; - vector regex_matches_dels; + vector diff_regex_matches_adds; + vector diff_regex_matches_dels; if (data->last_text_tokens.empty()) { additions = data->text; @@ -280,17 +315,17 @@ write_row(revisionData *data) if (!additions.empty()) { //cout << "ADD: " << additions << endl; - for (vector::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) { - pcrecpp::RE& regex = *r; - regex_matches_adds.push_back(regex.PartialMatch(additions)); + for (vector::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) { + pcrecpp::RE& diff_regex = *r; + diff_regex_matches_adds.push_back(diff_regex.PartialMatch(additions)); } } if (!deletions.empty()) { //cout << "DEL: " << deletions << endl; - for (vector::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) { - pcrecpp::RE& regex = *r; - regex_matches_dels.push_back(regex.PartialMatch(deletions)); + for (vector::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) { + pcrecpp::RE& diff_regex = *r; + diff_regex_matches_dels.push_back(diff_regex.PartialMatch(deletions)); } } @@ -309,15 +344,19 @@ write_row(revisionData *data) << data->editorid << "\t" << ((data->minor) ? "TRUE" : "FALSE") << "\t" << (unsigned int) data->text_size << "\t" - << shannon_H(data->text, data->text_size) << "\t" << md5_hex_output << "\t" << reverted_to << "\t" << (int) additions.size() << "\t" << (int) deletions.size(); - for (int n = 0; n < data->regex_names.size(); ++n) { - cout << "\t" << ((!regex_matches_adds.empty() && regex_matches_adds.at(n)) ? "TRUE" : "FALSE") - << "\t" << ((!regex_matches_dels.empty() && regex_matches_dels.at(n)) ? "TRUE" : "FALSE"); + for (int n = 0; n < data->content_regex_names.size(); ++n) { + cout << "\t" << ((!content_regex_matches.empty() + && content_regex_matches.at(n)) ? "TRUE" : "FALSE"); + } + + for (int n = 0; n < data->diff_regex_names.size(); ++n) { + cout << "\t" << ((!diff_regex_matches_adds.empty() && diff_regex_matches_adds.at(n)) ? "TRUE" : "FALSE") + << "\t" << ((!diff_regex_matches_dels.empty() && diff_regex_matches_dels.at(n)) ? "TRUE" : "FALSE"); } cout << endl; @@ -488,22 +527,26 @@ void print_usage(char* argv[]) { cerr << "usage: | " << argv[0] << "[options]" << endl << endl << "options:" << endl - << " -t print text and comments after each line of tab separated data" << endl - << " -n name of the following regex (e.g. -n name -r \"...\")" << endl - << " -r regex to check against additions and deletions" << endl + << " -v verbose mode prints text and comments after each line of tab separated data" << endl + << " -n name of the following regex for contet (e.g. -n name -r \"...\")" << endl + << " -r regex to check against content of the revision" << endl + << " -N name of the following regex for diffs (e.g. -N name -R \"...\")" << endl + << " -R regex to check against diffs (i.e., additions and deletions)" << endl + << " -t parse revisions only from pages whose titles match regex(es)" << endl << endl << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl << "a tab-separated stream of revisions on standard out:" << endl << endl << "title, articleid, revid, timestamp, anon, editor, editorid, minor," << endl - << "text_length, text_entropy, text_md5, reversion, additions_size, deletions_size" << endl + << "text_length, text_md5, reversion, additions_size, deletions_size" << endl << ".... and additional fields for each regex executed against add/delete diffs" << endl << endl << "Boolean fields are TRUE/FALSE except in the case of reversion, which is blank" << endl << "unless the article is a revert to a previous revision, in which case, it" << endl << "contains the revision ID of the revision which was reverted to." << endl << endl - << "author: Erik Garrison " << endl; + << "authors: Erik Garrison " << endl; + << " Benjamin Mako Hill " << endl; } @@ -516,34 +559,48 @@ main(int argc, char *argv[]) // in "simple" output, we don't print text and comments output_type = SIMPLE; char c; - string regex_name; + string diff_regex_name; + string content_regex_name; // the user data struct which is passed to callback functions revisionData data; - while ((c = getopt(argc, argv, "htn:r:")) != -1) + while ((c = getopt(argc, argv, "hvn:r:t:")) != -1) switch (c) { case 'd': dry_run = 1; break; - case 't': + case 'v': output_type = FULL; break; case 'n': - regex_name = optarg; + content_regex_name = optarg; break; case 'r': - data.regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); - data.regex_names.push_back(regex_name); - if (!regex_name.empty()) { - regex_name.clear(); + data.content_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + data.content_regex_names.push_back(content_regex_name); + if (!content_regex_name.empty()) { + content_regex_name.clear(); + } + break; + case 'N': + diff_regex_name = optarg; + break; + case 'R': + data.diff_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + data.diff_regex_names.push_back(diff_regex_name); + if (!diff_regex_name.empty()) { + diff_regex_name.clear(); } break; case 'h': print_usage(argv); exit(0); break; + case 't': + data.title_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + break; } if (dry_run) { // lets us print initialization options @@ -577,31 +634,42 @@ main(int argc, char *argv[]) cout << "title" << "\t" << "articleid" << "\t" << "revid" << "\t" - << "date" << " " + << "date" << "_" << "time" << "\t" << "anon" << "\t" << "editor" << "\t" << "editor_id" << "\t" << "minor" << "\t" << "text_size" << "\t" - << "text_entropy" << "\t" << "text_md5" << "\t" << "reversion" << "\t" << "additions_size" << "\t" << "deletions_size"; int n = 0; - if (!data.regexes.empty()) { - for (vector::iterator r = data.regexes.begin(); r != data.regexes.end(); ++r, ++n) { - if (data.regex_names.at(n).empty()) { + if (!data.content_regexes.empty()) { + for (vector::iterator r = data.content_regexes.begin(); + r != data.content_regexes.end(); ++r, ++n) { + if (data.content_regex_names.at(n).empty()) { + cout << "\t" << "regex" << n; + } else { + cout << "\t" << data.content_regex_names.at(n); + } + } + } + + if (!data.diff_regexes.empty()) { + for (vector::iterator r = data.diff_regexes.begin(); r != data.diff_regexes.end(); ++r, ++n) { + if (data.diff_regex_names.at(n).empty()) { cout << "\t" << "regex_" << n << "_add" << "\t" << "regex_" << n << "_del"; } else { - cout << "\t" << data.regex_names.at(n) << "_add" - << "\t" << data.regex_names.at(n) << "_del"; + cout << "\t" << data.diff_regex_names.at(n) << "_add" + << "\t" << data.diff_regex_names.at(n) << "_del"; } } } + cout << endl; // shovel data into the parser