#include <stdlib.h>
#include "expat.h"
#include <getopt.h>
-#include "disorder.h"
#include "md5.h"
#include "dtl/dtl.hpp"
#include <vector>
#define MEGABYTE 1048576
#define FIELD_BUFFER_SIZE 1024
-// 2048 KB in bytes + 1
-//#define TEXT_BUFFER_SIZE 2097153
-//#define TEXT_BUFFER_SIZE 10485760
+
+// this can be changed at runtime if we encounter an article larger than 10mb
+size_t text_buffer_size = 10 * MEGABYTE;
enum elements {
TITLE, ARTICLEID, REVISION, REVID, TIMESTAMP, CONTRIBUTOR,
char *comment;
char *text;
vector<string> last_text_tokens;
- vector<pcrecpp::RE> regexes;
- vector<string> regex_names;
+
+ // title regexes
+ vector<pcrecpp::RE> title_regexes;
+
+ // regexes for checking with revisions
+ vector<string> content_regex_names;
+ vector<pcrecpp::RE> content_regexes;
+
+ // regexes for looking within diffs
+ vector<string> diff_regex_names;
+ vector<pcrecpp::RE> diff_regexes;
+
map<string, string> revision_md5; // used for detecting reversions
// track string size of the elements, to prevent O(N^2) processing in charhndl
static void
init_data(revisionData *data, outtype output_type)
{
- data->text = (char*) malloc(4 * MEGABYTE); // 2MB is the article length limit, 4MB is 'safe'?
+ data->text = (char*) malloc(text_buffer_size);
data->comment = (char*) malloc(FIELD_BUFFER_SIZE);
data->title = (char*) malloc(FIELD_BUFFER_SIZE);
data->articleid = (char*) malloc(FIELD_BUFFER_SIZE);
++pos;
}
+ // look to see if (a) we've passed in a list of /any/ title_regexes
+ // and (b) if all of the title_regex_matches match
+ // if (a) is true and (b) is not, we return
+ bool any_title_regex_match = false;
+ if (!data->title_regexes.empty()) {
+ for (vector<pcrecpp::RE>::iterator r = data->title_regexes.begin(); r != data->title_regexes.end(); ++r) {
+ pcrecpp::RE& title_regex = *r;
+ if (title_regex.PartialMatch(data->title)) {
+ any_title_regex_match = true;
+ break;
+ }
+ }
+ if (!any_title_regex_match) {
+ return;
+ }
+ }
+
+ // search the content of the revision for a any of the regexes
+ vector<bool> content_regex_matches;
+ if (!data->content_regexes.empty()) {
+ for (vector<pcrecpp::RE>::iterator r = data->content_regexes.begin(); r != data->content_regexes.end(); ++r) {
+ pcrecpp::RE& content_regex = *r;
+ content_regex_matches.push_back(content_regex.PartialMatch(data->text));
+ }
+ }
+
//vector<string> additions;
//vector<string> deletions;
string additions;
string deletions;
- vector<bool> regex_matches_adds;
- vector<bool> regex_matches_dels;
+ vector<bool> diff_regex_matches_adds;
+ vector<bool> diff_regex_matches_dels;
- if (!data->last_text_tokens.empty()) {
+ if (data->last_text_tokens.empty()) {
+ additions = data->text;
+ } else {
// do the diff
dtl::Diff< string, vector<string> > d(data->last_text_tokens, text_tokens);
break;
}
}
-
- if (!additions.empty()) {
- //cout << "ADD: " << additions << endl;
- for (vector<pcrecpp::RE>::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) {
- pcrecpp::RE& regex = *r;
- regex_matches_adds.push_back(regex.PartialMatch(additions));
- }
+ }
+
+ if (!additions.empty()) {
+ //cout << "ADD: " << additions << endl;
+ for (vector<pcrecpp::RE>::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) {
+ pcrecpp::RE& diff_regex = *r;
+ diff_regex_matches_adds.push_back(diff_regex.PartialMatch(additions));
}
+ }
- if (!deletions.empty()) {
- //cout << "DEL: " << deletions << endl;
- for (vector<pcrecpp::RE>::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) {
- pcrecpp::RE& regex = *r;
- regex_matches_dels.push_back(regex.PartialMatch(deletions));
- }
+ if (!deletions.empty()) {
+ //cout << "DEL: " << deletions << endl;
+ for (vector<pcrecpp::RE>::iterator r = data->diff_regexes.begin(); r != data->diff_regexes.end(); ++r) {
+ pcrecpp::RE& diff_regex = *r;
+ diff_regex_matches_dels.push_back(diff_regex.PartialMatch(deletions));
}
-
- // apply regex to the diff
-
}
data->last_text_tokens = text_tokens;
<< data->editorid << "\t"
<< ((data->minor) ? "TRUE" : "FALSE") << "\t"
<< (unsigned int) data->text_size << "\t"
- << shannon_H(data->text, data->text_size) << "\t"
<< md5_hex_output << "\t"
<< reverted_to << "\t"
<< (int) additions.size() << "\t"
<< (int) deletions.size();
- for (int n = 0; n < data->regex_names.size(); ++n) {
- cout << "\t" << ((!regex_matches_adds.empty() && regex_matches_adds.at(n)) ? "TRUE" : "FALSE")
- << "\t" << ((!regex_matches_dels.empty() && regex_matches_dels.at(n)) ? "TRUE" : "FALSE");
+ for (int n = 0; n < data->content_regex_names.size(); ++n) {
+ cout << "\t" << ((!content_regex_matches.empty()
+ && content_regex_matches.at(n)) ? "TRUE" : "FALSE");
+ }
+
+ for (int n = 0; n < data->diff_regex_names.size(); ++n) {
+ cout << "\t" << ((!diff_regex_matches_adds.empty() && diff_regex_matches_adds.at(n)) ? "TRUE" : "FALSE")
+ << "\t" << ((!diff_regex_matches_dels.empty() && diff_regex_matches_dels.at(n)) ? "TRUE" : "FALSE");
}
cout << endl;
char*
strlcatn(char *dest, const char *src, size_t dest_len, size_t n)
{
- //size_t dest_len = strlen(dest);
size_t i;
for (i = 0 ; i < n && src[i] != '\0' ; i++)
charhndl(void* vdata, const XML_Char* s, int len)
{
revisionData* data = (revisionData*) vdata;
+ size_t bufsz;
if (data->element != UNUSED && data->position != SKIP) {
- //char t[len];
- //strncpy(t,s,len);
- //t[len] = '\0'; // makes t a well-formed string
switch (data->element) {
case TEXT:
- // printf("buffer length = %i, text: %s\n", len, t);
+ // check if we'd overflow our buffer
+ bufsz = data->text_size + len;
+ if (bufsz + 1 > text_buffer_size) {
+ data->text = (char*) realloc(data->text, bufsz + 1);
+ text_buffer_size = bufsz + 1;
+ }
strlcatn(data->text, s, data->text_size, len);
- data->text_size += len;
+ data->text_size = bufsz;
break;
case COMMENT:
strlcatn(data->comment, s, data->comment_size, len);
cerr << "usage: <wikimedia dump xml> | " << argv[0] << "[options]" << endl
<< endl
<< "options:" << endl
- << " -t print text and comments after each line of tab separated data" << endl
- << " -n name of the following regex (e.g. -N name -r \"...\")" << endl
- << " -r regex to check against additions and deletions" << endl
+ << " -v verbose mode prints text and comments after each line of tab separated data" << endl
+ << " -n name of the following regex for content (e.g. -n name -r \"...\")" << endl
+ << " -r regex to check against content of the revision" << endl
+ << " -N name of the following regex for diffs (e.g. -N name -R \"...\")" << endl
+ << " -R regex to check against diffs (i.e., additions and deletions)" << endl
+ << " -t parse revisions only from pages whose titles match regex(es)" << endl
<< endl
<< "Takes a wikimedia data dump XML stream on standard in, and produces" << endl
<< "a tab-separated stream of revisions on standard out:" << endl
<< endl
<< "title, articleid, revid, timestamp, anon, editor, editorid, minor," << endl
- << "text_length, text_entropy, text_md5, reversion, additions_size, deletions_size" << endl
+ << "text_length, text_md5, reversion, additions_size, deletions_size" << endl
<< ".... and additional fields for each regex executed against add/delete diffs" << endl
<< endl
- << "Notes:" << endl
<< "Boolean fields are TRUE/FALSE except in the case of reversion, which is blank" << endl
<< "unless the article is a revert to a previous revision, in which case, it" << endl
<< "contains the revision ID of the revision which was reverted to." << endl
<< endl
- << "author: Erik Garrison <erik@hypervolu.me>" << endl;
+ << "authors: Erik Garrison <erik@hypervolu.me>" << endl
+ << " Benjamin Mako Hill <mako@atdot.cc>" << endl;
}
// in "simple" output, we don't print text and comments
output_type = SIMPLE;
char c;
- string regex_name;
+ string diff_regex_name;
+ string content_regex_name;
// the user data struct which is passed to callback functions
revisionData data;
- while ((c = getopt(argc, argv, "htn:r:")) != -1)
+ while ((c = getopt(argc, argv, "hvn:r:t:")) != -1)
switch (c)
{
case 'd':
dry_run = 1;
break;
- case 't':
+ case 'v':
output_type = FULL;
break;
case 'n':
- regex_name = optarg;
+ content_regex_name = optarg;
break;
case 'r':
- data.regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
- data.regex_names.push_back(regex_name);
- if (!regex_name.empty()) {
- regex_name.clear();
+ data.content_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
+ data.content_regex_names.push_back(content_regex_name);
+ if (!content_regex_name.empty()) {
+ content_regex_name.clear();
+ }
+ break;
+ case 'N':
+ diff_regex_name = optarg;
+ break;
+ case 'R':
+ data.diff_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
+ data.diff_regex_names.push_back(diff_regex_name);
+ if (!diff_regex_name.empty()) {
+ diff_regex_name.clear();
}
break;
case 'h':
print_usage(argv);
exit(0);
break;
+ case 't':
+ data.title_regexes.push_back(pcrecpp::RE(optarg, pcrecpp::UTF8()));
+ break;
}
if (dry_run) { // lets us print initialization options
cout << "title" << "\t"
<< "articleid" << "\t"
<< "revid" << "\t"
- << "date" << " "
+ << "date" << "_"
<< "time" << "\t"
<< "anon" << "\t"
<< "editor" << "\t"
<< "editor_id" << "\t"
<< "minor" << "\t"
<< "text_size" << "\t"
- << "text_entropy" << "\t"
<< "text_md5" << "\t"
<< "reversion" << "\t"
<< "additions_size" << "\t"
<< "deletions_size";
int n = 0;
- if (!data.regexes.empty()) {
- for (vector<pcrecpp::RE>::iterator r = data.regexes.begin(); r != data.regexes.end(); ++r, ++n) {
- if (data.regex_names.at(n).empty()) {
+ if (!data.content_regexes.empty()) {
+ for (vector<pcrecpp::RE>::iterator r = data.content_regexes.begin();
+ r != data.content_regexes.end(); ++r, ++n) {
+ if (data.content_regex_names.at(n).empty()) {
+ cout << "\t" << "regex" << n;
+ } else {
+ cout << "\t" << data.content_regex_names.at(n);
+ }
+ }
+ }
+
+ if (!data.diff_regexes.empty()) {
+ for (vector<pcrecpp::RE>::iterator r = data.diff_regexes.begin(); r != data.diff_regexes.end(); ++r, ++n) {
+ if (data.diff_regex_names.at(n).empty()) {
cout << "\t" << "regex_" << n << "_add"
<< "\t" << "regex_" << n << "_del";
} else {
- cout << "\t" << data.regex_names.at(n) << "_add"
- << "\t" << data.regex_names.at(n) << "_del";
+ cout << "\t" << data.diff_regex_names.at(n) << "_add"
+ << "\t" << data.diff_regex_names.at(n) << "_del";
}
}
}
+
cout << endl;
// shovel data into the parser
// passes the buffer of data to the parser and checks for error
// (this is where the callbacks are invoked)
if (XML_Parse(parser, buf, len, done) == XML_STATUS_ERROR) {
- cerr << XML_ErrorString(XML_GetErrorCode(parser)) << " at line "
+ cerr << "XML ERROR: " << XML_ErrorString(XML_GetErrorCode(parser)) << " at line "
<< (int) XML_GetCurrentLineNumber(parser) << endl;
return 1;
}
projects / wikiq / blobdiff