X-Git-Url: https://projects.mako.cc/source/wikiq/blobdiff_plain/59363fd9d5c67c2ec370d436c329508c4b7a6f54..61ee7626135a5cd0b1379448cc967d3e9022852c:/wikiq.cpp diff --git a/wikiq.cpp b/wikiq.cpp index da5e7b7..0343dd3 100644 --- a/wikiq.cpp +++ b/wikiq.cpp @@ -56,8 +56,14 @@ typedef struct { char *comment; char *text; vector last_text_tokens; - vector regexes; - vector regex_names; + + // title regexes + vector title_regexes; + + // regexes for looking within diffs + vector diff_regex_names; + vector diff_regexes; + map revision_md5; // used for detecting reversions // track string size of the elements, to prevent O(N^2) processing in charhndl @@ -246,13 +252,30 @@ write_row(revisionData *data) ++pos; } + // look to see if (a) we've passed in a list of /any/ title_regexes + // and (b) if all of the title_regex_matches match + // if (a) is true and (b) is not, we return + bool any_title_regex_match = false; + if (!data->title_regexes.empty()) { + for (vector::iterator r = data->title_regexes.begin(); r != data->title_regexes.end(); ++r) { + pcrecpp::RE& title_regex = *r; + if (title_regex.PartialMatch(data->title)) { + any_title_regex_match = true; + break; + } + } + if (!any_title_regex_match) { + return; + } + } + //vector additions; //vector deletions; string additions; string deletions; - vector regex_matches_adds; - vector regex_matches_dels; + vector diff_regex_matches_adds; + vector diff_regex_matches_dels; if (data->last_text_tokens.empty()) { additions = data->text; @@ -280,17 +303,17 @@ write_row(revisionData *data) if (!additions.empty()) { //cout << "ADD: " << additions << endl; - for (vector::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) { - pcrecpp::RE& regex = *r; - regex_matches_adds.push_back(regex.PartialMatch(additions)); + for (vector::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) { + pcrecpp::RE& diff_regex = *r; + diff_regex_matches_adds.push_back(diff_regex.PartialMatch(additions)); } } if (!deletions.empty()) { //cout << "DEL: " << deletions << endl; - for (vector::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) { - pcrecpp::RE& regex = *r; - regex_matches_dels.push_back(regex.PartialMatch(deletions)); + for (vector::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) { + pcrecpp::RE& diff_regex = *r; + diff_regex_matches_dels.push_back(diff_regex.PartialMatch(deletions)); } } @@ -315,9 +338,9 @@ write_row(revisionData *data) << (int) additions.size() << "\t" << (int) deletions.size(); - for (int n = 0; n < data->regex_names.size(); ++n) { - cout << "\t" << ((!regex_matches_adds.empty() && regex_matches_adds.at(n)) ? "TRUE" : "FALSE") - << "\t" << ((!regex_matches_dels.empty() && regex_matches_dels.at(n)) ? "TRUE" : "FALSE"); + for (int n = 0; n < data->diff_regex_names.size(); ++n) { + cout << "\t" << ((!diff_regex_matches_adds.empty() && diff_regex_matches_adds.at(n)) ? "TRUE" : "FALSE") + << "\t" << ((!diff_regex_matches_dels.empty() && diff_regex_matches_dels.at(n)) ? "TRUE" : "FALSE"); } cout << endl; @@ -488,9 +511,10 @@ void print_usage(char* argv[]) { cerr << "usage: | " << argv[0] << "[options]" << endl << endl << "options:" << endl - << " -t print text and comments after each line of tab separated data" << endl - << " -n name of the following regex (e.g. -n name -r \"...\")" << endl - << " -r regex to check against additions and deletions" << endl + << " -v verbose mode prints text and comments after each line of tab separated data" << endl + << " -N name of the following regex for diffs (e.g. -N name -R \"...\")" << endl + << " -R regex to check against diffs (i.e., additions and deletions)" << endl + << " -t parse revisions only from pages whose titles match regex(es)" << endl << endl << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl << "a tab-separated stream of revisions on standard out:" << endl @@ -516,34 +540,37 @@ main(int argc, char *argv[]) // in "simple" output, we don't print text and comments output_type = SIMPLE; char c; - string regex_name; + string diff_regex_name; // the user data struct which is passed to callback functions revisionData data; - while ((c = getopt(argc, argv, "htn:r:")) != -1) + while ((c = getopt(argc, argv, "hvn:r:t:")) != -1) switch (c) { case 'd': dry_run = 1; break; - case 't': + case 'v': output_type = FULL; break; - case 'n': - regex_name = optarg; + case 'N': + diff_regex_name = optarg; break; - case 'r': - data.regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); - data.regex_names.push_back(regex_name); - if (!regex_name.empty()) { - regex_name.clear(); + case 'R': + data.diff_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + data.diff_regex_names.push_back(diff_regex_name); + if (!diff_regex_name.empty()) { + diff_regex_name.clear(); } break; case 'h': print_usage(argv); exit(0); break; + case 't': + data.title_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + break; } if (dry_run) { // lets us print initialization options @@ -591,14 +618,14 @@ main(int argc, char *argv[]) << "deletions_size"; int n = 0; - if (!data.regexes.empty()) { - for (vector::iterator r = data.regexes.begin(); r != data.regexes.end(); ++r, ++n) { - if (data.regex_names.at(n).empty()) { + if (!data.diff_regexes.empty()) { + for (vector::iterator r = data.diff_regexes.begin(); r != data.diff_regexes.end(); ++r, ++n) { + if (data.diff_regex_names.at(n).empty()) { cout << "\t" << "regex_" << n << "_add" << "\t" << "regex_" << n << "_del"; } else { - cout << "\t" << data.regex_names.at(n) << "_add" - << "\t" << data.regex_names.at(n) << "_del"; + cout << "\t" << data.diff_regex_names.at(n) << "_add" + << "\t" << data.diff_regex_names.at(n) << "_del"; } } }