From: Benjamin Mako Hill Date: Mon, 2 May 2011 19:56:48 +0000 (-0400) Subject: added support for regex title matching X-Git-Url: https://projects.mako.cc/source/wikiq/commitdiff_plain/8e1c1afcc765c0c7df502b5451e6035e7272c8c6?ds=inline added support for regex title matching Added support for regex matching of titles to (for example) match only things from particular namespaces. In the process, I changed the "t" option which prints out text to "d" for debug and used the t option for taking title regexes. --- diff --git a/wikiq.cpp b/wikiq.cpp index da5e7b7..e8ca4d2 100644 --- a/wikiq.cpp +++ b/wikiq.cpp @@ -57,6 +57,7 @@ typedef struct { char *text; vector last_text_tokens; vector regexes; + vector wp_namespace_res; vector regex_names; map revision_md5; // used for detecting reversions @@ -246,6 +247,22 @@ write_row(revisionData *data) ++pos; } + // skip this if the wp_namespace is not in the proscribed list of + // namespaces + bool wp_namespace_found = false; + if (!data->wp_namespace_res.empty()) { + for (vector::iterator r = data->wp_namespace_res.begin(); r != data->wp_namespace_res.end(); ++r) { + pcrecpp::RE& wp_namespace_re = *r; + if (wp_namespace_re.PartialMatch(data->title)) { + wp_namespace_found = true; + break; + } + } + if (!wp_namespace_found) { + return; + } + } + //vector additions; //vector deletions; string additions; @@ -488,9 +505,10 @@ void print_usage(char* argv[]) { cerr << "usage: | " << argv[0] << "[options]" << endl << endl << "options:" << endl - << " -t print text and comments after each line of tab separated data" << endl + << " -v verbose mode prints text and comments after each line of tab separated data" << endl << " -n name of the following regex (e.g. -n name -r \"...\")" << endl << " -r regex to check against additions and deletions" << endl + << " -t regex(es) to check title against as a way of limiting output to specific namespaces" << endl << endl << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl << "a tab-separated stream of revisions on standard out:" << endl @@ -521,13 +539,13 @@ main(int argc, char *argv[]) // the user data struct which is passed to callback functions revisionData data; - while ((c = getopt(argc, argv, "htn:r:")) != -1) + while ((c = getopt(argc, argv, "hvn:r:t:")) != -1) switch (c) { case 'd': dry_run = 1; break; - case 't': + case 'v': output_type = FULL; break; case 'n': @@ -544,6 +562,9 @@ main(int argc, char *argv[]) print_usage(argv); exit(0); break; + case 't': + data.wp_namespace_res.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + break; } if (dry_run) { // lets us print initialization options