lower case N in the usage instructions

[wikiq] / wikiq.cpp

diff --git a/wikiq.cpp b/wikiq.cpp
index bd895d4314a5b99184ecfd5d42914ad8099da1b7..da5e7b74d50c484242f1731d016140822b8e12cc 100644 (file)
--- a/wikiq.cpp
+++ b/wikiq.cpp
@@ -28,9 +28,9 @@ using namespace std;
 
 #define MEGABYTE 1048576
 #define FIELD_BUFFER_SIZE 1024
 
 #define MEGABYTE 1048576
 #define FIELD_BUFFER_SIZE 1024

-// 2048 KB in bytes + 1
-//#define TEXT_BUFFER_SIZE 2097153
-//#define TEXT_BUFFER_SIZE 10485760
+
+// this can be changed at runtime if we encounter an article larger than 10mb
+size_t text_buffer_size = 10 * MEGABYTE;

 
 enum elements { 
     TITLE, ARTICLEID, REVISION, REVID, TIMESTAMP, CONTRIBUTOR, 
 
 enum elements { 
     TITLE, ARTICLEID, REVISION, REVID, TIMESTAMP, CONTRIBUTOR, 


@@ -162,7 +162,7 @@ void cleanup_article(revisionData *data) {
 static void 
 init_data(revisionData *data, outtype output_type)
 {
 static void 
 init_data(revisionData *data, outtype output_type)
 {

-    data->text = (char*) malloc(4 * MEGABYTE);  // 2MB is the article length limit, 4MB is 'safe'?
+    data->text = (char*) malloc(text_buffer_size);

     data->comment = (char*) malloc(FIELD_BUFFER_SIZE);
     data->title = (char*) malloc(FIELD_BUFFER_SIZE);
     data->articleid = (char*) malloc(FIELD_BUFFER_SIZE);
     data->comment = (char*) malloc(FIELD_BUFFER_SIZE);
     data->title = (char*) malloc(FIELD_BUFFER_SIZE);
     data->articleid = (char*) malloc(FIELD_BUFFER_SIZE);


@@ -254,7 +254,9 @@ write_row(revisionData *data)
     vector<bool> regex_matches_adds;
     vector<bool> regex_matches_dels;
 
     vector<bool> regex_matches_adds;
     vector<bool> regex_matches_dels;
 

-    if (!data->last_text_tokens.empty()) {
+    if (data->last_text_tokens.empty()) {
+        additions = data->text;
+    } else {

         // do the diff
         
         dtl::Diff< string, vector<string> > d(data->last_text_tokens, text_tokens);
         // do the diff
         
         dtl::Diff< string, vector<string> > d(data->last_text_tokens, text_tokens);


@@ -274,25 +276,22 @@ write_row(revisionData *data)
                 break;
             }
         }
                 break;
             }
         }

-
-        if (!additions.empty()) {
-            //cout << "ADD: " << additions << endl;
-            for (vector<pcrecpp::RE>::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) {
-                pcrecpp::RE& regex = *r;
-                regex_matches_adds.push_back(regex.PartialMatch(additions));
-            }
+    }
+    
+    if (!additions.empty()) {
+        //cout << "ADD: " << additions << endl;
+        for (vector<pcrecpp::RE>::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) {
+            pcrecpp::RE& regex = *r;
+            regex_matches_adds.push_back(regex.PartialMatch(additions));

         }
         }

+    }

 
 

-        if (!deletions.empty()) {
-            //cout << "DEL: " << deletions << endl;
-            for (vector<pcrecpp::RE>::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) {
-                pcrecpp::RE& regex = *r;
-                regex_matches_dels.push_back(regex.PartialMatch(deletions));
-            }
+    if (!deletions.empty()) {
+        //cout << "DEL: " << deletions << endl;
+        for (vector<pcrecpp::RE>::iterator r = data->regexes.begin(); r != data->regexes.end(); ++r) {
+            pcrecpp::RE& regex = *r;
+            regex_matches_dels.push_back(regex.PartialMatch(deletions));

         }
         }

-
-        // apply regex to the diff
-

     }
 
     data->last_text_tokens = text_tokens;
     }
 
     data->last_text_tokens = text_tokens;


@@ -343,7 +342,6 @@ split_timestamp(revisionData *data)
 char*
 strlcatn(char *dest, const char *src, size_t dest_len, size_t n)
 {
 char*
 strlcatn(char *dest, const char *src, size_t dest_len, size_t n)
 {

-   //size_t dest_len = strlen(dest);

    size_t i;
 
    for (i = 0 ; i < n && src[i] != '\0' ; i++)
    size_t i;
 
    for (i = 0 ; i < n && src[i] != '\0' ; i++)


@@ -357,15 +355,18 @@ static void
 charhndl(void* vdata, const XML_Char* s, int len)
 { 
     revisionData* data = (revisionData*) vdata;
 charhndl(void* vdata, const XML_Char* s, int len)
 { 
     revisionData* data = (revisionData*) vdata;

+    size_t bufsz;

     if (data->element != UNUSED && data->position != SKIP) {
     if (data->element != UNUSED && data->position != SKIP) {

-        //char t[len];
-        //strncpy(t,s,len);
-        //t[len] = '\0'; // makes t a well-formed string

         switch (data->element) {
             case TEXT:
         switch (data->element) {
             case TEXT:

-                   // printf("buffer length = %i, text: %s\n", len, t);
+                    // check if we'd overflow our buffer
+                    bufsz = data->text_size + len;
+                    if (bufsz + 1 > text_buffer_size) {
+                        data->text = (char*) realloc(data->text, bufsz + 1);
+                        text_buffer_size = bufsz + 1;
+                    }

                     strlcatn(data->text, s, data->text_size, len);
                     strlcatn(data->text, s, data->text_size, len);

-                    data->text_size += len;
+                    data->text_size = bufsz;

                     break;
             case COMMENT:
                     strlcatn(data->comment, s, data->comment_size, len);
                     break;
             case COMMENT:
                     strlcatn(data->comment, s, data->comment_size, len);


@@ -488,7 +489,7 @@ void print_usage(char* argv[]) {
          << endl
          << "options:" << endl
          << "  -t   print text and comments after each line of tab separated data" << endl
          << endl
          << "options:" << endl
          << "  -t   print text and comments after each line of tab separated data" << endl

-         << "  -n   name of the following regex (e.g. -N name -r \"...\")" << endl
+         << "  -n   name of the following regex (e.g. -n name -r \"...\")" << endl

          << "  -r   regex to check against additions and deletions" << endl
          << endl
          << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl
          << "  -r   regex to check against additions and deletions" << endl
          << endl
          << "Takes a wikimedia data dump XML stream on standard in, and produces" << endl


@@ -613,7 +614,7 @@ main(int argc, char *argv[])
         // passes the buffer of data to the parser and checks for error
         //   (this is where the callbacks are invoked)
         if (XML_Parse(parser, buf, len, done) == XML_STATUS_ERROR) {
         // passes the buffer of data to the parser and checks for error
         //   (this is where the callbacks are invoked)
         if (XML_Parse(parser, buf, len, done) == XML_STATUS_ERROR) {

-            cerr << XML_ErrorString(XML_GetErrorCode(parser)) << " at line "
+            cerr << "XML ERROR: " << XML_ErrorString(XML_GetErrorCode(parser)) << " at line "

                  << (int) XML_GetCurrentLineNumber(parser) << endl;
             return 1;
         }
                  << (int) XML_GetCurrentLineNumber(parser) << endl;
             return 1;
         }