X-Git-Url: https://projects.mako.cc/source/wikiq/blobdiff_plain/95cc4c318cf4e6e46b8ae9e947799eee9d30a8c9..21341e70f9d509aeaaf0cb6a39caf56d704dc838:/wikiq.cpp diff --git a/wikiq.cpp b/wikiq.cpp index 6635dc0..ad9ac48 100644 --- a/wikiq.cpp +++ b/wikiq.cpp @@ -11,7 +11,6 @@ #include #include "expat.h" #include -#include "disorder.h" #include "md5.h" #include "dtl/dtl.hpp" #include @@ -28,9 +27,9 @@ using namespace std; #define MEGABYTE 1048576 #define FIELD_BUFFER_SIZE 1024 -// 2048 KB in bytes + 1 -//#define TEXT_BUFFER_SIZE 2097153 -//#define TEXT_BUFFER_SIZE 10485760 + +// this can be changed at runtime if we encounter an article larger than 10mb +size_t text_buffer_size = 10 * MEGABYTE; enum elements { TITLE, ARTICLEID, REVISION, REVID, TIMESTAMP, CONTRIBUTOR, @@ -56,8 +55,18 @@ typedef struct { char *comment; char *text; vector last_text_tokens; - vector regexes; - vector regex_names; + + // title regexes + vector title_regexes; + + // regexes for checking with revisions + vector content_regex_names; + vector content_regexes; + + // regexes for looking within diffs + vector diff_regex_names; + vector diff_regexes; + map revision_md5; // used for detecting reversions // track string size of the elements, to prevent O(N^2) processing in charhndl @@ -162,7 +171,7 @@ void cleanup_article(revisionData *data) { static void init_data(revisionData *data, outtype output_type) { - data->text = (char*) malloc(4 * MEGABYTE); // 2MB is the article length limit, 4MB is 'safe'? + data->text = (char*) malloc(text_buffer_size); data->comment = (char*) malloc(FIELD_BUFFER_SIZE); data->title = (char*) malloc(FIELD_BUFFER_SIZE); data->articleid = (char*) malloc(FIELD_BUFFER_SIZE); @@ -246,15 +255,43 @@ write_row(revisionData *data) ++pos; } + // look to see if (a) we've passed in a list of /any/ title_regexes + // and (b) if all of the title_regex_matches match + // if (a) is true and (b) is not, we return + bool any_title_regex_match = false; + if (!data->title_regexes.empty()) { + for (vector::iterator r = data->title_regexes.begin(); r != data->title_regexes.end(); ++r) { + pcrecpp::RE& title_regex = *r; + if (title_regex.PartialMatch(data->title)) { + any_title_regex_match = true; + break; + } + } + if (!any_title_regex_match) { + return; + } + } + + // search the content of the revision for a any of the regexes + vector content_regex_matches; + if (!data->content_regexes.empty()) { + for (vector::iterator r = data->content_regexes.begin(); r != data->content_regexes.end(); ++r) { + pcrecpp::RE& content_regex = *r; + content_regex_matches.push_back(content_regex.PartialMatch(data->text)); + } + } + //vector additions; //vector deletions; string additions; string deletions; - vector regex_matches_adds; - vector regex_matches_dels; + vector diff_regex_matches_adds; + vector diff_regex_matches_dels; - if (!data->last_text_tokens.empty()) { + if (data->last_text_tokens.empty()) { + additions = data->text; + } else { // do the diff dtl::Diff< string, vector > d(data->last_text_tokens, text_tokens); @@ -274,25 +311,22 @@ write_row(revisionData *data) break; } } - - if (!additions.empty()) { - //cout << "ADD: " << additions << endl; - for (vector::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) { - pcrecpp::RE& regex = *r; - regex_matches_adds.push_back(regex.PartialMatch(additions)); - } + } + + if (!additions.empty()) { + //cout << "ADD: " << additions << endl; + for (vector::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) { + pcrecpp::RE& diff_regex = *r; + diff_regex_matches_adds.push_back(diff_regex.PartialMatch(additions)); } + } - if (!deletions.empty()) { - //cout << "DEL: " << deletions << endl; - for (vector::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) { - pcrecpp::RE& regex = *r; - regex_matches_dels.push_back(regex.PartialMatch(deletions)); - } + if (!deletions.empty()) { + //cout << "DEL: " << deletions << endl; + for (vector::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) { + pcrecpp::RE& diff_regex = *r; + diff_regex_matches_dels.push_back(diff_regex.PartialMatch(deletions)); } - - // apply regex to the diff - } data->last_text_tokens = text_tokens; @@ -310,15 +344,19 @@ write_row(revisionData *data) << data->editorid << "\t" << ((data->minor) ? "TRUE" : "FALSE") << "\t" << (unsigned int) data->text_size << "\t" - << shannon_H(data->text, data->text_size) << "\t" << md5_hex_output << "\t" << reverted_to << "\t" << (int) additions.size() << "\t" << (int) deletions.size(); - for (int n = 0; n < data->regex_names.size(); ++n) { - cout << "\t" << ((!regex_matches_adds.empty() && regex_matches_adds.at(n)) ? "TRUE" : "FALSE") - << "\t" << ((!regex_matches_dels.empty() && regex_matches_dels.at(n)) ? "TRUE" : "FALSE"); + for (int n = 0; n < data->content_regex_names.size(); ++n) { + cout << "\t" << ((!content_regex_matches.empty() + && content_regex_matches.at(n)) ? "TRUE" : "FALSE"); + } + + for (int n = 0; n < data->diff_regex_names.size(); ++n) { + cout << "\t" << ((!diff_regex_matches_adds.empty() && diff_regex_matches_adds.at(n)) ? "TRUE" : "FALSE") + << "\t" << ((!diff_regex_matches_dels.empty() && diff_regex_matches_dels.at(n)) ? "TRUE" : "FALSE"); } cout << endl; @@ -343,7 +381,6 @@ split_timestamp(revisionData *data) char* strlcatn(char *dest, const char *src, size_t dest_len, size_t n) { - //size_t dest_len = strlen(dest); size_t i; for (i = 0 ; i < n && src[i] != '\0' ; i++) @@ -357,15 +394,18 @@ static void charhndl(void* vdata, const XML_Char* s, int len) { revisionData* data = (revisionData*) vdata; + size_t bufsz; if (data->element != UNUSED && data->position != SKIP) { - //char t[len]; - //strncpy(t,s,len); - //t[len] = '\0'; // makes t a well-formed string switch (data->element) { case TEXT: - // printf("buffer length = %i, text: %s\n", len, t); + // check if we'd overflow our buffer + bufsz = data->text_size + len; + if (bufsz + 1 > text_buffer_size) { + data->text = (char*) realloc(data->text, bufsz + 1); + text_buffer_size = bufsz + 1; + } strlcatn(data->text, s, data->text_size, len); - data->text_size += len; + data->text_size = bufsz; break; case COMMENT: strlcatn(data->comment, s, data->comment_size, len); @@ -487,23 +527,26 @@ void print_usage(char* argv[]) { cerr << "usage: | " << argv[0] << "[options]" << endl << endl << "options:" << endl - << " -t print text and comments after each line of tab separated data" << endl - << " -n name of the following regex (e.g. -N name -r \"...\")" << endl - << " -r regex to check against additions and deletions" << endl + << " -v verbose mode prints text and comments after each line of tab separated data" << endl + << " -n name of the following regex for content (e.g. -n name -r \"...\")" << endl + << " -r regex to check against content of the revision" << endl + << " -N name of the following regex for diffs (e.g. -N name -R \"...\")" << endl + << " -R regex to check against diffs (i.e., additions and deletions)" << endl + << " -t parse revisions only from pages whose titles match regex(es)" << endl << endl << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl << "a tab-separated stream of revisions on standard out:" << endl << endl << "title, articleid, revid, timestamp, anon, editor, editorid, minor," << endl - << "text_length, text_entropy, text_md5, reversion, additions_size, deletions_size" << endl + << "text_length, text_md5, reversion, additions_size, deletions_size" << endl << ".... and additional fields for each regex executed against add/delete diffs" << endl << endl - << "Notes:" << endl << "Boolean fields are TRUE/FALSE except in the case of reversion, which is blank" << endl << "unless the article is a revert to a previous revision, in which case, it" << endl << "contains the revision ID of the revision which was reverted to." << endl << endl - << "author: Erik Garrison " << endl; + << "authors: Erik Garrison " << endl + << " Benjamin Mako Hill " << endl; } @@ -516,34 +559,48 @@ main(int argc, char *argv[]) // in "simple" output, we don't print text and comments output_type = SIMPLE; char c; - string regex_name; + string diff_regex_name; + string content_regex_name; // the user data struct which is passed to callback functions revisionData data; - while ((c = getopt(argc, argv, "htn:r:")) != -1) + while ((c = getopt(argc, argv, "hvn:r:t:")) != -1) switch (c) { case 'd': dry_run = 1; break; - case 't': + case 'v': output_type = FULL; break; case 'n': - regex_name = optarg; + content_regex_name = optarg; break; case 'r': - data.regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); - data.regex_names.push_back(regex_name); - if (!regex_name.empty()) { - regex_name.clear(); + data.content_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + data.content_regex_names.push_back(content_regex_name); + if (!content_regex_name.empty()) { + content_regex_name.clear(); + } + break; + case 'N': + diff_regex_name = optarg; + break; + case 'R': + data.diff_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + data.diff_regex_names.push_back(diff_regex_name); + if (!diff_regex_name.empty()) { + diff_regex_name.clear(); } break; case 'h': print_usage(argv); exit(0); break; + case 't': + data.title_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8())); + break; } if (dry_run) { // lets us print initialization options @@ -577,31 +634,42 @@ main(int argc, char *argv[]) cout << "title" << "\t" << "articleid" << "\t" << "revid" << "\t" - << "date" << " " + << "date" << "_" << "time" << "\t" << "anon" << "\t" << "editor" << "\t" << "editor_id" << "\t" << "minor" << "\t" << "text_size" << "\t" - << "text_entropy" << "\t" << "text_md5" << "\t" << "reversion" << "\t" << "additions_size" << "\t" << "deletions_size"; int n = 0; - if (!data.regexes.empty()) { - for (vector::iterator r = data.regexes.begin(); r != data.regexes.end(); ++r, ++n) { - if (data.regex_names.at(n).empty()) { + if (!data.content_regexes.empty()) { + for (vector::iterator r = data.content_regexes.begin(); + r != data.content_regexes.end(); ++r, ++n) { + if (data.content_regex_names.at(n).empty()) { + cout << "\t" << "regex" << n; + } else { + cout << "\t" << data.content_regex_names.at(n); + } + } + } + + if (!data.diff_regexes.empty()) { + for (vector::iterator r = data.diff_regexes.begin(); r != data.diff_regexes.end(); ++r, ++n) { + if (data.diff_regex_names.at(n).empty()) { cout << "\t" << "regex_" << n << "_add" << "\t" << "regex_" << n << "_del"; } else { - cout << "\t" << data.regex_names.at(n) << "_add" - << "\t" << data.regex_names.at(n) << "_del"; + cout << "\t" << data.diff_regex_names.at(n) << "_add" + << "\t" << data.diff_regex_names.at(n) << "_del"; } } } + cout << endl; // shovel data into the parser @@ -614,7 +682,7 @@ main(int argc, char *argv[]) // passes the buffer of data to the parser and checks for error // (this is where the callbacks are invoked) if (XML_Parse(parser, buf, len, done) == XML_STATUS_ERROR) { - cerr << XML_ErrorString(XML_GetErrorCode(parser)) << " at line " + cerr << "XML ERROR: " << XML_ErrorString(XML_GetErrorCode(parser)) << " at line " << (int) XML_GetCurrentLineNumber(parser) << endl; return 1; }