1 # EDIT ME: Configuration options are in the stanza below
2 #######################################################################
3 redirect.data.dir <- "output/redir"
4 spells.data.dir <- "output/spells"
5 output.data.dir <- "output"
7 # functions and code used elsewhere
8 #######################################################################
14 # options(mc.cores = 20)
15 # Sys.setenv(GOTO_NUM_THREADS=05)
17 generate.spells <- function (page, d) {
19 x <- d[page,mult="all"]
21 x <- x[sort.list(x$timestamp),]
23 # transform the target because there are some differences that don't matter
24 x$target <- gsub('_', ' ', x$target)
25 x$target <- gsub("(^[[:alpha:]])", "\\U\\1", x$target, perl=TRUE)
26 x$target <- gsub('\\#.*$', '', x$target)
29 x$redirect.prev <- c(FALSE, x$redirect[1:(length(x$redirect)-1)])
30 x$target.prev <- c(NA, x$target[1:(length(x$redirect)-1)])
32 x$redirect.prev <- FALSE
36 # get a list of transitions
37 x <- x[x$redirect != x$redirect.prev |
38 ((!is.na(x$target) & !is.na(x$target.prev)) &
39 x$target != x$target.prev),]
41 # if there is only one transition it stays that way
43 x$end <- c(x$timestamp[2:dim(x)[1]], NA)
48 x <- x[x$redirect == TRUE,]
51 x <- x[,c("page.id", "timestamp", "end", "page.title", "target")]
52 colnames(x) <- c("page.id", "start", "end", "page.title", "target")
57 filename.to.spells <- function (filename) {
58 con <- pipe(paste("bzcat", filename))
60 d <- read.delim(con, stringsAsFactors=FALSE, header=FALSE, skip=1,
61 encoding="UTF-8", quote="")
63 colnames(d) <- c("page.id", "revision.id", "page.title", "timestamp",
64 "deleted", "redirect", "target")
66 d$timestamp <- as.POSIXct(d$timestamp, tz="UTC", origin="1970-01-01 00:00:00")
70 redirected.pages <- unique(d$page.title[d$redirect])
72 # convert to data.table
74 setkey(d, "page.title")
76 redirect.spells <- rbindlist(lapply(redirected.pages, generate.spells, d))
78 return(redirect.spells)