X-Git-Url: https://projects.mako.cc/source/wikiq/blobdiff_plain/811f321153159f2fcfb093ffc43845a7315a336d..4ea02e87e6e73828bf7a6037c45c0b602998f15e:/wikiq.c

diff --git a/wikiq.c b/wikiq.c
index 4f7aa4d..9455086 100644
--- a/wikiq.c
+++ b/wikiq.c
@@ -17,6 +17,10 @@
 #define TIME_LENGTH 8
 #define TIMESTAMP_LENGTH 20
 
+// 2048 KB in bytes + 1
+#define TEXT_BUFFER_SIZE 2097153
+#define FIELD_BUFFER_SIZE 1024
+
 enum elements { 
     TITLE, ARTICLEID, REVISION, REVID, TIMESTAMP, CONTRIBUTOR, 
     EDITOR, EDITORID, MINOR, COMMENT, UNUSED, TEXT
@@ -24,26 +28,23 @@ enum elements {
 
 enum block { TITLE_BLOCK, REVISION_BLOCK, CONTRIBUTOR_BLOCK, SKIP };
 
-enum outtype { NORMAL, SIMPLE };
+enum outtype { FULL, SIMPLE };
 
 typedef struct {
 
-    struct {
-        char *title;
-        char *articleid;
-        char *revid;
-        char *date;
-        char *time;
-        char *timestamp;
-        char *anon;
-        char *editor;
-        char *editorid;
-        char *minor;
-        char *comment;
-        char *text;
-    } rev;
+    char *title;
+    char *articleid;
+    char *revid;
+    char *date;
+    char *time;
+    char *timestamp;
+    char *anon;
+    char *editor;
+    char *editorid;
+    bool minor;
+    char *comment;
+    char text[TEXT_BUFFER_SIZE];
     
-    char *dropstr;
     enum elements element;
     enum block position;
     enum outtype output_type;
@@ -60,19 +61,19 @@ static void
 clean_data(revisionData *data, int title)
 {
     if (title) {
-        data->rev.title = NULL;
-        data->rev.articleid = NULL;
+        data->title = NULL;
+        data->articleid = NULL;
     }
-    data->rev.revid = NULL;
-    data->rev.date = NULL;
-    data->rev.time = NULL;
-    data->rev.timestamp = NULL;
-    data->rev.anon = NULL;
-    data->rev.editor = NULL;
-    data->rev.editorid = NULL;
-    data->rev.minor = NULL;
-    data->rev.comment = NULL; 
-    data->rev.text = NULL;
+    data->revid = NULL;
+    data->date = NULL;
+    data->time = NULL;
+    data->timestamp = NULL;
+    data->anon = NULL;
+    data->editor = NULL;
+    data->editorid = NULL;
+    data->minor = false;
+    data->comment = NULL; 
+    //data->text = NULL;
     data->element = UNUSED;
     //data->position = 
 }
@@ -82,37 +83,36 @@ free_data(revisionData *data, int title)
 {
     if (title) {
         //printf("freeing article\n");
-        free(data->rev.title);
-        free(data->rev.articleid);
+        free(data->title);
+        free(data->articleid);
     }
-    free(data->rev.revid);
-    free(data->rev.date);
-    free(data->rev.time);
-    free(data->rev.timestamp);
-    free(data->rev.anon);
-    free(data->rev.editor);
-    free(data->rev.editorid);
-    free(data->rev.minor);
-    free(data->rev.comment);
-    free(data->rev.text);
+    free(data->revid);
+    free(data->date);
+    free(data->time);
+    free(data->timestamp);
+    free(data->anon);
+    free(data->editor);
+    free(data->editorid);
+    free(data->comment);
+    //free(data->text);
+    data->text[0] = '\0';
 }
 
-cleanup_revision(revisionData *data) {
+void cleanup_revision(revisionData *data) {
     free_data(data, 0);
     clean_data(data, 0);
 }
 
-cleanup_article(revisionData *data) {
+void cleanup_article(revisionData *data) {
     free_data(data, 1);
     clean_data(data, 1);
 }
 
 
 static void 
-init_data(revisionData *data, char *dropstr, int output_type)
+init_data(revisionData *data, outtype output_type)
 {
     clean_data(data, 1); // sets every element to null...
-    data->dropstr = dropstr;
     data->output_type = output_type;
 }
 
@@ -123,17 +123,17 @@ print_state(revisionData *data)
 {
     printf("element = %i\n", data->element);
     printf("output_type = %i\n", data->output_type);
-    printf("title = %s\n", data->rev.title);
-    printf("articleid = %s\n", data->rev.articleid);
-    printf("revid = %s\n", data->rev.revid);
-    printf("date = %s\n", data->rev.date);
-    printf("time = %s\n", data->rev.time);
-    printf("anon = %s\n", data->rev.anon);
-    printf("editor = %s\n", data->rev.editor);
-    printf("editorid = %s\n", data->rev.editorid);
-    printf("minor = %s\n", data->rev.minor);
-    printf("comment = %s\n", data->rev.comment); 
-    printf("text = %s\n", data->rev.text);
+    printf("title = %s\n", data->title);
+    printf("articleid = %s\n", data->articleid);
+    printf("revid = %s\n", data->revid);
+    printf("date = %s\n", data->date);
+    printf("time = %s\n", data->time);
+    printf("anon = %s\n", data->anon);
+    printf("editor = %s\n", data->editor);
+    printf("editorid = %s\n", data->editorid);
+    printf("minor = %s\n", (data->minor ? "1" : "0"));
+    printf("comment = %s\n", data->comment); 
+    printf("text = %s\n", data->text);
     printf("\n");
 
 }
@@ -160,77 +160,33 @@ write_header()
 static void
 write_row(revisionData *data)
 {
-    // define temporary variables to hold output values:
-    char *title, *articleid; 
-    char *revid, *date, *time, *anon, *editor, *editorid;
-    char *minor, *comment;
-    char *text;
-    // perform some simple logic to obtain correct output values
-
-    if (data->rev.minor == NULL)
-        minor = "0";
-    else minor = data->rev.minor;
-
-    if (data->rev.editor == NULL)
-        anon = "1";
-    else anon = "0";
-
-    if (data->rev.title ==  NULL)
-        title = "";
-    else title = data->rev.title;
-
-    if (data->rev.articleid == NULL)
-        articleid = "";
-    else articleid = data->rev.articleid;
-
-    if (data->rev.revid == NULL)
-        revid = "";
-    else revid = data->rev.revid;
-
-    if (data->rev.date == NULL)
-        date = "";
-    else date = data->rev.date;
-
-    if (data->rev.time == NULL)
-        time = "";
-    else time = data->rev.time;
-
-    if (data->rev.editor == NULL)
-        editor = "";
-    else editor = data->rev.editor;
-    
-    if (data->rev.editorid == NULL)
-        editorid = "";
-    else editorid = data->rev.editorid;
-
-    if (data->rev.text == NULL)
-        text = "";
-    else text = data->rev.text;
-
-    
-    if (data->rev.comment == NULL)
-        comment = "";
-    else comment = data->rev.comment;
-    
 
     // TODO: make it so you can specify fields to output
     // note that date and time are separated by a space, to match postgres's 
     // timestamp format
+    printf("%s\t%s\t%s\t%s %s\t%s\t%s\t%s\t%s",
+        (data->title != NULL) ? data->title : "",
+        (data->articleid != NULL) ? data->articleid : "",
+        (data->revid != NULL) ? data->revid : "",
+        (data->date != NULL) ? data->date : "",
+        (data->time != NULL) ? data->time : "",
+        (data->editor != NULL) ? "0" : "1",
+        (data->editor != NULL) ? data->editor : "",
+        (data->editorid != NULL) ? data->editorid  : "",
+        (data->minor) ? "1" : "0");
     switch (data->output_type)
     {
-        case NORMAL:
-            printf("%s\t%s\t%s\t%s %s\t%s\t%s\t%s\t%s\t%s\t%s\n",
-                title,articleid,revid,date,time,anon,editor,editorid,minor,comment,text);
-            break;
         case SIMPLE:
-            printf("%s\t%s\t%s\t%s %s\t%s\t%s\t%s\t%s\n",
-                title,articleid,revid,date,time,anon,editor,editorid,minor);
+            printf("\t%i\n", (unsigned int) strlen(data->text));
+            break;
+        case FULL:
+            printf("\t%s\t%s\n", data->comment, data->text);
             break;
     }
 
 }
 
-static char
+void
 *timestr(char *timestamp, char time_buffer[TIME_LENGTH+1])
 {
     char *timeinstamp = &timestamp[DATE_LENGTH+1];
@@ -239,7 +195,7 @@ static char
 }
 
 
-static char
+void
 *datestr(char *timestamp, char date_buffer[DATE_LENGTH+1])
 {
     strncpy(date_buffer, timestamp, DATE_LENGTH);
@@ -247,49 +203,49 @@ static char
 }
 
 char
-*append(char *entry, char *new)
+*append(char *entry, char *newstr)
 {
     char *newbuff;
     int len;
-    len = (strlen(entry)+strlen(new))*sizeof(char) + 1;
-    newbuff = realloc(entry, len);
-    strcat(newbuff, new);
+    len = (strlen(entry)+strlen(newstr))*sizeof(char) + 1;
+    newbuff = (char*) realloc(entry, len);
+    strcat(newbuff, newstr);
     return newbuff;
 }
 
 char
-*cache(char *entry, char *new)
+*cache(char *entry, char *newstr)
 {
     char *newbuff;
     int len;
-    len = strlen(new)*sizeof(char) + 1; // include space for the '\0' !
-    newbuff = malloc(len);
-    strcpy(newbuff,new);
+    len = strlen(newstr)*sizeof(char) + 1; // include space for the '\0' !
+    newbuff = (char*) malloc(len);
+    strcpy(newbuff,newstr);
     return newbuff;
 
 }
 
 char
-*store(char *entry, char *new)
+*store(char *entry, char *newstr)
 {
     char *newbuff;
     if (entry == NULL)
-        newbuff = cache(entry, new);
+        newbuff = cache(entry, newstr);
     else 
-        newbuff = append(entry, new);
+        newbuff = append(entry, newstr);
     return newbuff;
 }
 
 void
 split_timestamp(revisionData *data) 
 {
-    char *t = data->rev.timestamp;
+    char *t = data->timestamp;
     char date_buffer[DATE_LENGTH+1];
     char time_buffer[TIME_LENGTH+1];
     datestr(t, date_buffer);
     timestr(t, time_buffer);
-    data->rev.date = store(data->rev.date, date_buffer);
-    data->rev.time = store(data->rev.time, time_buffer);
+    data->date = store(data->date, date_buffer);
+    data->time = store(data->time, time_buffer);
 }
 
 /* currently unused */
@@ -306,29 +262,9 @@ is_whitespace(char *string) {
 }
 
 static void
-squeeze(char *s, int c) {
-    int i, j;
-    for (i = j = 0; s[i] != '\0'; i++)
-        if (s[i] != c)
-            s[j++] = s[i];
-    s[j] = '\0';
-}
-
-int
-contains(char *s, char *t)
-{
-    char c = t[0]; //just get the first character of t
-    int i = 0;
-    while (s[i] != '\0') {
-        if (s[i] == c) 
-            return 1;
-        i++;
-    }
-}
-
-static void
-charhndl(revisionData *data, char *s, int len)
+charhndl(void* vdata, const XML_Char* s, int len)
 { 
+    revisionData* data = (revisionData*) vdata;
     if (data->element != UNUSED && data->position != SKIP) {
         char t[len];
         strncpy(t,s,len);
@@ -336,34 +272,30 @@ charhndl(revisionData *data, char *s, int len)
         switch (data->element) {
             case TITLE:
                 {
-                    data->rev.title = store(data->rev.title, t);
+                    data->title = store(data->title, t);
                     // skip any articles with bad characters in their titles
-                    if (contains(t, data->dropstr)) {
-                        data->position = SKIP;
-                        //printf("found a baddie\n");
-                    }
                     break;
                 }
             case ARTICLEID:
                    // printf("articleid = %s\n", t);
-                    data->rev.articleid = store(data->rev.articleid, t);
+                    data->articleid = store(data->articleid, t);
                     break;
             case REVID:
                    // printf("revid = %s\n", t);
-                    data->rev.revid = store(data->rev.revid, t);
+                    data->revid = store(data->revid, t);
                     break;
             case TIMESTAMP: 
-                    data->rev.timestamp = store(data->rev.timestamp, t); 
-                    if (strlen(data->rev.timestamp) == TIMESTAMP_LENGTH)
+                    data->timestamp = store(data->timestamp, t); 
+                    if (strlen(data->timestamp) == TIMESTAMP_LENGTH)
                         split_timestamp(data);
                     break;
             case EDITOR: {
-                    data->rev.editor = store(data->rev.editor, t);
+                    data->editor = store(data->editor, t);
                     break;
                     }
             case EDITORID: 
                     //printf("editorid = %s\n", t);
-                    data->rev.editorid = store(data->rev.editorid, t);
+                    data->editorid = store(data->editorid, t);
                     break;
             /* the following are implied or skipped:
             case MINOR: 
@@ -373,10 +305,16 @@ charhndl(revisionData *data, char *s, int len)
             */
             case COMMENT: 
                    // printf("row: comment is %s\n", t);
-                    data->rev.comment = store(data->rev.comment, t);
+                    //if (data->output_type == FULL) {
+                        data->comment = store(data->comment, t);
+                    //}
                     break;
             case TEXT:
-                   data->rev.text = store(data->rev.text, t);
+                    //if (data->output_type == FULL) {
+                        //data->text = store(data->text, t);
+                        //
+                    strcat(data->text, t);
+                    //}
                    break; 
             default: break;
         }
@@ -384,8 +322,9 @@ charhndl(revisionData *data, char *s, int len)
 }
 
 static void
-start(revisionData *data, const char *name, const char **attr)
+start(void* vdata, const XML_Char* name, const XML_Char** attr)
 {
+    revisionData* data = (revisionData*) vdata;
     
     if (strcmp(name,"title") == 0) {
         cleanup_article(data); // cleans up data from last article
@@ -414,7 +353,7 @@ start(revisionData *data, const char *name, const char **attr)
         // minor tag has no character data, so we parse here
         else if (strcmp(name,"minor") == 0) {
             data->element = MINOR;
-            data->rev.minor = store(data->rev.minor, "1"); 
+            data->minor = true; 
         }
         else if (strcmp(name,"timestamp") == 0)
             data->element = TIMESTAMP;
@@ -442,8 +381,9 @@ start(revisionData *data, const char *name, const char **attr)
 
 
 static void
-end(revisionData *data, const char *name)
+end(void* vdata, const XML_Char* name)
 {
+    revisionData* data = (revisionData*) vdata;
     if (strcmp(name, "revision") == 0 && data->position != SKIP) {
         write_row(data); // crucial... :)
         cleanup_revision(data);  // also crucial
@@ -472,24 +412,20 @@ int
 main(int argc, char *argv[])
 {
     
-    char *dropstr = "";
     enum outtype output_type;
     int dry_run = 0;
     // in "simple" output, we don't print text and comments
     output_type = SIMPLE;
     char c;
 
-    while ((c = getopt(argc, argv, "hr:sd")) != -1)
+    while ((c = getopt(argc, argv, "ht")) != -1)
         switch (c)
         {
-            case 'r':
-                dropstr = optarg;
-                break;
             case 'd':
                 dry_run = 1;
                 break;
             case 't':
-                output_type = NORMAL;
+                output_type = FULL;
                 break;
             case 'h':
                 print_usage(argv);
@@ -499,7 +435,6 @@ main(int argc, char *argv[])
 
     if (dry_run) { // lets us print initialization options
         printf("simple_output = %i\n", output_type);
-        printf("dropstr = %s\n", dropstr);
         exit(1);
     }
 
@@ -509,15 +444,19 @@ main(int argc, char *argv[])
     // initialize the user data struct which is passed to callback functions
     revisionData data;  
     // initialize the elements of the struct to default values
-    init_data(&data, dropstr, output_type);
+    init_data(&data, output_type);
 
 
     // makes the parser pass "data" as the first argument to every callback 
     XML_SetUserData(parser, &data);
+    void (*startFnPtr)(void*, const XML_Char*, const XML_Char**) = start;
+    void (*endFnPtr)(void*, const XML_Char*) = end;
+    void (*charHandlerFnPtr)(void*, const XML_Char*, int) = charhndl;
+
     // sets start and end to be the element start and end handlers
-    XML_SetElementHandler(parser, (void *) start, (void *) end);
+    XML_SetElementHandler(parser, startFnPtr, endFnPtr);
     // sets charhndl to be the callback for raw character data
-    XML_SetCharacterDataHandler(parser, (void *) charhndl);
+    XML_SetCharacterDataHandler(parser, charHandlerFnPtr);
 
     int done;
     char buf[BUFSIZ];