imported the talk given at wikimania

[state_of_wikimedia_research_2014] / 20140808-wikimania_research.tex

\documentclass[xcolor=dvipsnames]{beamer}

% set up the file to create notes in the output PDFs
\usepackage{pgfpages}
\input{notes.config}

\renewcommand{\rmdefault}{ugm}
\usepackage[garamond]{mathdesign}

\renewcommand{\sfdefault}{phv}
\usepackage{relsize}

\usepackage{ucs}
\usepackage[utf8x]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{textcomp}

% add tikz and a bunch of tikz foo
\usepackage{tikz}
\usetikzlibrary{shapes,shapes.misc,backgrounds,fit,positioning}
\tikzstyle{every picture}+=[overlay,remember picture]

% add functions to circle parts of slides (e.g., in tables)
\newcommand\marktopleft[1]{%
    \tikz[overlay,remember picture] 
        \node (marker-#1-a) at (0,1.5ex) {};%
}
\newcommand\markbottomright[1]{%
    \tikz[overlay,remember picture] 
        \node (marker-#1-b) at (0,0) {};%
    \tikz[overlay,remember picture,dashed,inner sep=3pt]
        \node[violet!75,ultra thick,draw,rounded rectangle,fit=(marker-#1-a.center) (marker-#1-b.center)] {};%
}

% DEPRECATED function to build a huge centered dropshadow
\newcommand\dropshadow[3]{%
  \node[black!30!white] at (#1+0.1,#2-0.1) {
    \scalebox{2}{\Huge \textbf{#3}}
  };
  \node at (#1,#2) {
    \scalebox{2}{\Huge \e{#3}}
  };
}

% create an empty quotetxt so we can reuse it
\newcommand{\quotetxt}{}

% more flexible non-tikz alternative with no dropshadow 
\newlength{\centertxtlen}
\makeatletter
\newcommand\centertext[2]{%
  \setlength{\centertxtlen}{#1}%
  \setlength{\centertxtlen}{0.48\centertxtlen}%
  {\centering
    \fontsize{#1}{2\centertxtlen}\selectfont
    \e{#2}

  }
}

% add function to stop numbering appendix slides
\newcommand{\backupbegin}{
   \newcounter{framenumberappendix}
   \setcounter{framenumberappendix}{\value{framenumber}}
}
\newcommand{\backupend}{
   \addtocounter{framenumberappendix}{-\value{framenumber}}
   \addtocounter{framenumber}{\value{framenumberappendix}} 
}

% packages i use in essentially every document
\usepackage{graphicx}
\usepackage{url}
% \usepackage{dcolumn}
% \usepackage{booktabs}

% replace footnotes with symbols instead of numbers
\renewcommand*{\thefootnote}{\fnsymbol{footnote}}
\usepackage{perpage}
\MakePerPage{footnote}

%% BEAMER THEME STUFF
\usetheme[pageofpages=/,% String used between the current page and the
                         % total page count.
          bullet=default,% Use circles instead of squares for bullets.
          titleline=false,% Show a line below the frame title.
          alternativetitlepage=true,% Use the fancy title page.
          titlepagelogo=figures/logo.pdf,% Logo for the first page.
          %watermark=watermark-polito,% Watermark used in every page.
          watermarkheight=100px,% Height of the watermark.
          watermarkheightmult=4,% The watermark image is 4 times bigger
                                % than watermarkheight.
          ]{Torino}

\usecolortheme{mako}
\useinnertheme{rectangles}
%\setbeamertemplate{blocks}[rounded][]
\setbeamercolor{block title}{bg=makopurple3, fg=White}

\setbeamertemplate{items}[default] 
\setbeamertemplate{blocks}[shadow=true] 

\usepackage{tcolorbox}
% These options will be applied to all `tcolorboxes`
\tcbset{%
    noparskip,
    colback=makopurple5, %background color of the box
    colframe=makopurple1, %color of frame and title background
    coltext=black, %color of body text
    coltitle=white, %color of title text 
    arc=0em,
    left=0.1em,
    right=0.1em,
    fonttitle=\bfseries,
    alerted/.style={coltitle=red, 
                     colframe=gray!40},
    example/.style={coltitle=black, 
                     colframe=green!20,             
                     colback=green!5},
    }


%\useoutertheme{infolines}


\hypersetup{colorlinks=true, linkcolor=Black, citecolor=Black, filecolor=makopurple1,
    urlcolor=Plum, unicode=true}

% create a boldface version of the header
\setbeamerfont{frametitle}{series=\bfseries}
\setbeamerfont{title}{series=\bfseries}

% tweak the beamer font to make it a bit lists a bit smaller
\setbeamerfont*{itemize/enumerate body}{size=\small}
\setbeamerfont*{itemize/enumerate subbody}{size=\footnotesize}
\setbeamerfont*{itemize/enumerate subsubbody}{size=\footnotesize}

% indent the margins of the itemize lists a little bit
\setlength{\leftmargin}{0pt}
\setlength{\leftmargini}{0.7cm}
\setlength{\leftmarginii}{0.7cm}

% create a new \e{} command to make things purple and bold
\newcommand{\e}[1]{\textcolor{makopurple1}{\textbf{#1}}}

% remove the nagivation symbols
\setbeamertemplate{navigation symbols}{}

\title{Presentation Title}
% \subtitle{Presentation Subtitle}
\author[Benj. Mako Hill]{\textbf{Benjamin Mako Hill}\\ mako@mit.edu}

\institute[MIT/Harvard]{\textbf{Massachusetts Institute of Technology}\\
  Sloan School of Management\\
  MIT Media Lab\\
  \hspace{1pt}\\
  \textbf{Harvard University}\\
  Berkman Center for Internet and Society}

\date{December 2, 1980}

\begin{document}

% remove some of the space in the itemize to make it quite compact
\let\olditemize\itemize
\renewcommand\itemize{\olditemize\itemsep-1pt}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%% SLIDE: Title Slide
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]

  \begin{tikzpicture}
    \node at (current page.center) [xshift=-3.5cm, yshift=0.5cm, opacity=0.4]
      {\includegraphics[height=\paperheight]{figures/wikimedia_projects.png}};


    \node at (current page.south east)
    [anchor=south east,text width=1.8\paperwidth,align=right,color=black]
    {
      {\spaceskip 0.3em%
        \fontsize{2.5em}{2.5em}
        \selectfont {\bf \color{makopurple4} The State of Wikimedia\\
          Research: 2013-2014} \par}

      \vspace{1em}

      {\spaceskip 0.3em%
        \fontsize{2.0em}{2.1em}
        \selectfont {\bf \color{black} Benjamin Mako Hill\\
          Tilman Bayer\\
          Aaron Shaw\\
          Wikimania 2014, London\\
          August 8, 2014} \par}

    };
  \end{tikzpicture}
  
  \input{vc}

  \tikz[overlay,shift=(current page.south west)]{\node [xshift=5.6em,yshift=0.5em]{\colorbox{makopurple1}{\color{white} \tt \smaller \smaller \smaller revision:\ \VCRevision\  (\VCDateTEX)}};}

  \note{I've been doing this for many years. I started in 2008 and
    have done this almost every single year since.

    This began as an excuse for me to make sure I was up to date on
    Wikimedia Research.}

\end{frame}

%% SLIDE: Anecdote from Wikimania 2008
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\renewcommand{\quotetxt}{``This talk will try to [provide] a quick
  tour – a literature review in the scholarly parlance – of the last
  year's academic landscape around Wikimedia and its projects geared
  at non-academic editors and readers. It will try to categorize,
  distill, and describe, from a birds eye view, the academic landscape
  as it is shaping up around
  our project.''\\
  \hfill – \e{From my Wikimania 2008 Submission}}

\begin{frame}

  {\smaller \quotetxt}

  \pause
  \includegraphics[width=\textwidth]{figures/google_scholar_result.png}

  \pause
  \tikz{\draw (current page.center) [xshift=-2.1cm, yshift=0.9cm, color=red]
    ellipse (1.5cm and 0.5cm);}

  \note<1>{Back in Wikimania 2008, I set out to run a session at
    Wikimania that would provide a comprehensive literature review of
    articles in Wikipedia published in the last year.

    \begin{quote}
      \quotetxt
    \end{quote}

    Then, about two weeks before Wikimania, I did the scholar search
    so I could build the literature.}

  \note<2->{I tried to import the whole list into Zotero and managed
    to get banned for abusing the Google Scholar because they thought
    that no human being could realistically consume the amount of
    material published on Wikipedia that year.

    So anyway, I had a 45 minute talk so it worked out to 3.45 seconds
    to per paper...

    And believe it or not, this year is even bigger.
 
    And my talk is even shorter.}

\end{frame}

%% SLIDE: Citations Per Year
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}

  \includegraphics[width=\textwidth]{figures/citations_by_year.pdf}

  \centering

  {\smaller \emph{Number of citation, per year, with the term
      “wikipedia” in the title.\\
      (Source: Google scholar results. Accessed: 2013-08-06)}}

  \note{Academics have written \e{a lot} of papers about
    Wikipedia. There are more than 500 papers published about
    Wikipedia each year and although we've reached and moved past a
    peak it seems, it's not slowing by much.}

\end{frame}

\begin{frame}

  \begin{itemize}
    \larger \larger
  \item \e{2968} Wikipedia-related publications in the Scopus database
    as of November 2013

  \item \e{160} recent publications reviewed or mentioned in the 12 issues
    of the Wikimedia Research Newsletter August 2013-July 2014.
  \end{itemize}

\end{frame}

%% SLIDE: My Scope Conditions
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}

  \includegraphics[width=\textwidth]{figures/multiple_issues.png}

  \larger \larger
  In selecting papers for this session, the goal is always to choose
  examples of work that:

  \begin{itemize}
  \larger \larger
  \item Represent \e{important themes} from Wikipedia in the last year.
  \item Research that is likely to be of \e{interest} to Wikimedians.
  \item Research by people who are \e{not at Wikimania}.
  \end{itemize}

  \note{This is my disclaimer slide...

  Within these goals, the selections are \e{incomplete}, and \e{wrong}.}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Paper Summaries}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Event Prediction}

\begin{frame}
  \centertext{6em}{Event Prediction}

  \note{Mako

    This was the year that studies of readership of Wikipedia really
    blossomed. People figured out how to use the view data. Much of
    what they used it for was prediction.}
\end{frame}

\begin{frame}
  
  \frametitle{Wikipedia Viewership and Flu Prediction}

  \larger \larger McIver, David J., and John
  S. Brownstein. ``\e{Wikipedia Usage Estimates Prevalence of
    Influenza-Like Illness in the United States in Near Real-Time}.''
  PLoS Comput Biol 10, no. 4 (April 17, 2014):
  e1003581. \href{http://dx.doi.org/10.1371/journal.pcbi.1003581}{doi:10.1371/journal.pcbi.1003581}.

\end{frame}

\begin{frame}

  \frametitle{Wikipedia Viewership and Flu Prediction: Motivation}

  \begin{itemize}
    \larger \larger
  \item \e{Google Flu Trends} uses search engine queries to try to
    predict influenza epidemics more quickly than traditional methods.
  \item ..but it has been criticized as being biased (e.g., by media coverage).
  \item WP is freely available and viewership data is free, unlike
    Google which is proprietary.

  \end{itemize}

  \note{2009 H1N1 Swine Flu broke GFT.}
\end{frame}

\begin{frame}

  \frametitle{Wikipedia Viewership and Flu Prediction: Methods}

  \begin{itemize}
  \larger \larger \larger
  \item Measure traffic to flu related articles on Wikipedia
  \item Compare to the ``gold standard'' data from the Center for
    Disease Control (CDC)
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Wikipedia Viewership and Flu Prediction: Results}

  \centering
  \includegraphics[width=\textwidth]{figures/flu.png}

\note{\begin{itemize}
  \larger \larger

  \item Wikipedia better than Google at predicting peak flu weeks.
  \item Wikipedia better at predicting relative influenza rates.
  \end{itemize}}

\end{frame}

\begin{frame}

  \frametitle{Other things people have tried to predict include...}
  \begin{itemize}

    \larger \larger
  \item \href{http://arxiv.org/abs/1405.3612}{Global disease forecasting}

  \item \href{https://meta.wikimedia.org/wiki/Research:Newsletter/2014/June\#.22Prediction_of_Foreign_Box_Office_Revenues_Based_on_Wikipedia_Page_Activity.22}{Box office revenue based on films}
  \item \href{https://meta.wikimedia.org/wiki/Research:Newsletter/2013/December\#Attempt_to_use_Wikipedia_pageviews_to_predict_election_results_in_Iran.2C_Germany_and_the_UK}{Election results in Iran, Germany and the UK}
  \item \href{https://meta.wikimedia.org/wiki/Research:Newsletter/2014/April\#cite_ref-10}{Breaking news stories}
  \item Trending topics, general zeitgeist. 
    \href{https://meta.wikimedia.org/wiki/Research:Newsletter/2014/June\#cite_ref-13}{[1]}
    \href{https://meta.wikimedia.org/wiki/Research:Newsletter/2013/August\#Collective_memories_in_Wikipedia}{[2]}
    \href{https://meta.wikimedia.org/wiki/Research:Newsletter/2013/November\#Twitter_activity_leads_Wikipedia_activity_by_an_hour}{[3]}
    \href{https://meta.wikimedia.org/wiki/Research:Newsletter/2013/September\#Identifying_trending_topics_of_yesteryear}{[4]}

 \end{itemize}
  
\end{frame}

\subsection{Wikimedia as a Corpus}

\begin{frame}
  
  \centertext{5em}{Wikimedia as a Corpus}

  \note{\e{Aaron}

    From the Bar-Ilan lit review: 48\% of them about Wikipedia per se,
    52\% are just using Wikipedia (e.g. as a text corpus)
    
    Wiktionary, non Wikipedia projects.

    Wiktionary as a source of data, not the substance/object of
    analysis. Projects as amazing multilingual corpuses of natural
    language.}

\end{frame}

\begin{frame}

  \frametitle{Multi-Lingual Dictionary from Wiktionary: Methods}

  \larger \larger Ács, Judit. ``\e{Pivot-Based Multilingual Dictionary
    Building Using Wiktionary}.'' In Proceedings of the Ninth
  International Conference on Language Resources and Evaluation
  (LREC'14). Rekyjavik, Iceland,
  2014. \href{http://www.lrec-conf.org/proceedings/lrec2014/pdf/864\_Paper.pdf}{http://www.lrec-conf.org/proceedings/lrec2014/pdf/864\_Paper.pdf}.

  \note{\e{Methods}: This paper uses a machine classifier to take
    advantage of ``pivots'' --- words with common translations across
    more than one language --- to automatically construct a
    multilingual dictionary via triangulation! They then manually
    evaluate the precision of this multilingual dictionary on a small
    subset of words.}

\end{frame}

\begin{frame}

  \frametitle{Multi-Lingual Dictionary from Wiktionary: Results}


  \begin{columns}
    \column{0.5\textwidth}
    \includegraphics[height=0.85\textheight]{figures/Acs-fig5.pdf}

    \column{0.5\textwidth}

    \larger \e{Figure:} Translation graph with many pivots. The edge
    labels denote the source Wiktionary and article of the translation
    pair.

  \end{columns}

  \note{\textbf{Finding:} So, kind of incredibly, this sort of
    works. The author succeeds in constructing the multilingual
    dictionary, but finds that problems like polysemy (one word
    meaning multiple things) limit the precision of the resulting
    output. Using the multilingual pivots offered by Wiktionary,
    however, performs much better using ``triangles'' between three
    languages alone.}

\end{frame}

\subsection{Content Quality}
\begin{frame}
  
  \centertext{6em}{Content Quality}

  \note{Mako

    Mostly focused on sub-areas. There was one this year we considered
    discussion on hematology. Or information on infectious diseases.}

\end{frame}

\begin{frame}

  \frametitle{Comparing Wikipedia Quality to Britannica}
  
  \larger \larger Nifrário Rodrigues, Fernando
  Silvério. ``\e{Colaboração Em Massa Ou Amadorismo Em Massa? Um
    Estudo Comparativo Da Qualidade Da Informação Científica Produzida
    Utilizando Os Conceitos E Ferramentas Wiki}.'' Universidade de
  Évora,
  2012. \href{http://massamateurism.blogspot.co.uk/p/synopsis.html}{http://massamateurism.blogspot.co.uk/p/synopsis.html}.

  \note{A Portuguese-language dissertation at the Universidade de
    Évora, titled "Colaboração em Massa ou Amadorismo em Massa?"
    ("Mass collaboration or mass amateurism?")}

\end{frame}

\begin{frame}

  \frametitle{Comparing Wikipedia Quality to Britannica: Methods}

  \begin{itemize}
    \larger \larger
  \item Random sample of 245 article pairs from both encyclopedias.
  \item Graded by an expert in its subject area using a five-point scale.
  \item Experts asked, ``to concentrate only on some [...] intrinsic
    aspects of the articles' quality, namely accuracy and objectivity,
    and discard the contextual, representational and accessibility
    aspects.''
  \item Experts were mostly university teachers.
  \end{itemize}

  \note{Compared the quality of English Wikipedia with that of
    Encyclopedia Britannica}
\end{frame}

\begin{frame}
  \frametitle{Comparing Wikipedia Quality to Britannica: Results}

  \includegraphics[width=\textwidth]{figures/synopsis4.jpg}

  \note{They rated "\e{90\%} of the Wikipedia articles ... as having
    \e{equivalent or better quality} than their Britannica
    counterparts".}
\end{frame}

\begin{frame}

  \frametitle{Comparing Wikipedia Quality to Britannica: Results}

  \includegraphics[width=\textwidth]{figures/synopsis2.jpg}

\end{frame}

% Other examples:
% - https://meta.wikimedia.org/wiki/Research:Newsletter/2014/April#cite_ref-17
% - https://meta.wikimedia.org/wiki/Research:Newsletter/2014/February#.22World.E2.80.99s_largest_study_on_Wikipedia:_Better_than_its_reputation.22 (Finnish WP)
% - https://meta.wikimedia.org/wiki/Research:Newsletter/2014/March#.22Risk_factors_and_control_of_hospital_acquired_infections:_a_comparison_between_Wikipedia_and_scientific_literature.22
% - https://meta.wikimedia.org/wiki/Research:Newsletter/2013/December#.22Evaluation_of_gastroenterology_and_hepatology_articles_on_Wikipedia.22

\subsection{Controversy and Conflict}
\begin{frame}

  \centertext{7em}{Controversy and Conflict}

  \note{Tilman}

\end{frame}

\begin{frame}
  \frametitle{Conflict, Consensus and Quality in Wikipedia}
  
  \larger \larger Osman, Kim. ``\e{The Role of Conflict in Determining
    Consensus on Quality in Wikipedia Articles}.'' In Proceedings of
  the 9th International Symposium on Open Collaboration,
  12:1–12:6. WikiSym ’13. New York, NY, USA: ACM,
  2013. \href{http://dx.doi.org/10.1145/2491055.2491067}{doi:10.1145/2491055.2491067}.

\end{frame}

\begin{frame}

  \frametitle{Conflict, Consensus and Quality in Wikipedia: Methods}
  
  \begin{itemize}
    \larger \larger \larger
  \item Literature review of research on online communities
  \item Discourse analysis of [[Talk:Australia]] (+ 17 archive pages),
    coding 147 threads (156,112 words) in a grounded theory approach.
  \end{itemize}

  \note{Very simplified: Grounded theory is an approach in social
    sciences where one starts from empirical data first and develops
    hypotheses by coding... }

\end{frame}

\begin{frame}

  \frametitle{Conflict, Consensus and Quality in Wikipedia: Results}

  \larger \larger

  ``\e{Conflict} was significantly more prevalent .. than \e{collaboration}.''

  \begin{itemize}
    \larger

  \item However: Personal attacks are rare (as opposed to
    disagreement).

  \item ``The four main themes that emerged as cause for
    debate among the editorial community were \e{sources}, \e{wording},
    \e{structure} and \e{content accuracy}".

  \item In e.g. sourcing debates, "conflict ... had a role in developing a
    mechanism to ensure the accuracy of information by prompting
    participants to properly source and reference material."

  \item Conflict is not always bad. ``\e{Generative friction}''
    benefits Wikipedia quality.
  \end{itemize}

  \note{i.e. editors generally play the ball, not the man.}

\end{frame}

\begin{frame}
  
  \frametitle{Conflict, Consensus and Quality in Wikipedia: Results}

    \includegraphics[width=0.85\textwidth]{figures/osman-fig.png}

    \begin{itemize}
      \larger
    \item Talk page discussions frequently contain references to
      Wikipedia policy, both formal (directly linking a policy) and
      informal (mentioning or quoting it).

    \item 86\% of policy references are informal 

    \end{itemize}

  \note{Bear in mind that while it was a sizable corpus, it still
    pertained to only one article on the English Wikipedia.

    Osman: "More than being a set of isolated rules for the community,
    policies are part of the fabric of the culture of the talk
    pages. They regulate both behavior and the production process and
    manage conflict so that it remains a generative friction", e.g. by
    discouraging personal attacks.}

\end{frame}

\subsection{WikiProjects}

\begin{frame}
  
  \centertext{7em}{WikiProjects}

  \note{Another area of research focuses on understanding wikis and
    other peer production communities as organization. Some of the
    most interesting work in this area compares many projects in order
    to better understand the characteristics that might lead them to
    grow and attract contributors.}
\end{frame}

\begin{frame}
  \frametitle{Critical Mass in WikiProjects}

  \larger \larger Solomon, Jacob, and Rick Wash. “\e{Critical Mass of
    What? Exploring Community Growth in WikiProjects}.” In Eighth
  International AAAI Conference on Weblogs and Social Media,
  2014. \href{http://www.aaai.org/ocs/index.php/ICWSM/ICWSM14/paper/view/8104}{[1]}.

  \note{\e{Methods:} Analyzed data on 1069 EN:WP WikiProjects. Fit
    models to describe the projects' growth curves in terms of editors
    and project edits. They then use these growth curves to estimate
    the relationship between adding editors or edits (at an early
    stage) on subsequent project growth.}

\end{frame}

\begin{frame}
  \frametitle{Critical Mass in WikiProjects: Results}

  \includegraphics[width=\textwidth]{figures/Solomon_Walsh-2014-critical_mass_wikiprojects-FIG1.pdf}

  \note{Results: The authors find that projects with more contributors
    are more likely to experience subsequent growth in contributions
    and contributors.They also find that contributions from both
    "power users" and more casual, one-off contributors predicts
    subsequent growth. These findings (like others in this area) have
    important implications for project leaders and designers.}
  
\end{frame}

\subsection{Vandalism}
\begin{frame}

  \centertext{7em}{Vandalism}

\end{frame}

\begin{frame}

  \frametitle{Edit Patterns and Vandalism Detection}
  
  \larger \larger Sethi, Deepika. \e{A Large Scale Study of Edit
    Patterns in Wikipedia and its Applications to Vandalism
    Detection}. M. Sc. thesis, University of Georgia. Submitted
  December 2012

\end{frame}

\begin{frame}

  \frametitle{Edit Patterns and Vandalism Detection: Methods}  

  \larger \larger Used the \e{PAN Wikipedia vandalism corpus 2010 } of
  32,452 edits, classified as vandalism (2,391 edits) or non-vandalism
  by Mechanical Turk workers.

  \note{Corpus was created to train vandalism detectors and
    formed the basis of several competitions.}

  Identified vandalism based on:

  \begin{itemize}
  \larger \larger 

  \item Editor country
  \item Time of day

  \item \e{Article content domain} using 12 ``classes'' from DBpedia
    (e.g., 1. Person, 2. Work, 3. Sports, 4. Places, 5. Food ...)
  \item Content of edits

  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Edit Patterns and Vandalism Detection: Results}

  \larger \larger \larger "\e{Vandalism occurs the most during office
    hours} while \e{non-vandalism occurs the most during late
    evenings}."

\end{frame}

\begin{frame}
  \frametitle{Edit Patterns and Vandalism Detection: Results}

    \larger "Hostilities among the countries are one major cause of
    vandalism."
    
    \medskip

  \includegraphics[width=\textwidth]{figures/sethi-countries.pdf}
  \note{\e{Country} where the vandal is based..

    Methodology: Get Geolocation of IP editors to articles about
    countries Result: image : Figure 20: India’s Vandal Contributions
    (caption: Where Indians vandalize most)

    [[India]] most frequently vandalized from India, Bahrain, Bangladesh, Pakistan, ...

    [[Pakistan]] most frequently vandalized from India, Pakistan, Poland, Sweden}


\note{Other example: [[Taiwan]] from Taiwan, China, Germany ... / [[China]] from Ireland, Germany, Poland, ... }

\end{frame}

\begin{frame}
  \frametitle{Edit Patterns and Vandalism Detection: Results}

%  \includegraphics[width=\textwidth]{figures/sethi-46-figure.pdf}

  \larger \larger \larger \e{Common Vandalism Words}

  Ball, chicken, British, woman, hole, handicap, meat, kiss, play,
    old, love, death, course, kick, American, bomb

  \note{Methodology: Word most frequently occurring in vandalized versions

    Result: Ball, chicken, British, woman, hole, handicap, meat, kiss,
    play, old, love, death, course, kick, American, bomb, ...}


  %\note{Intriguing... A vandal who only added words might aim to get visibility for them; a vandal who only deleted words might dislike a particular statement}


\end{frame}

\subsection{Editor Motivation}

\begin{frame}

  \centertext{7em}{Editor Motivation}

\end{frame}

\begin{frame}

  \frametitle{When do barnstars increase edits?}

  \larger \larger Restivo, Michael, and Arnout van de Rijt. “\e{No Praise
  without Effort: Experimental Evidence on How Rewards Affect
  Wikipedia’s Contributor Community}.” Information, Communication \&
  Society 0, no. 0 (0):
  1–12. \href{http://dx.doi.org/10.1080/1369118X.2014.888459}{doi:10.1080/1369118X.2014.888459}.

\end{frame}

\begin{frame}

  \frametitle{When do barnstars increase edits? Methods}

  \larger \larger Prior work by the authors showed that randomly given
  barnstars to very active editors (top 1\% in a month) resulted in
  more edits and a positive feedback loop in English Wikipedia.

  \begin{itemize}
    \larger \larger
  \item Like before, gives barnstars to \e{randomly} selected users --
    but this time to editors of varying activity level (e.g.,
    91-95$^{\mathrm{th}}$, 96-99$^{\mathrm{th}}$, and 100$^{th}$
    percentile of editing in the month). Plus a ``control'' group of
    other editors who do not receive the award.
  \item Follow post-award activity on Wikipedia.
  \end{itemize}

\end{frame}

\begin{frame}

  \frametitle{Do barnstars encourage editing? Results for Edits}

  \includegraphics[width=\textwidth]{figures/barnstars-1.pdf}

  \note{Positive effect only in the case of the very active -- top 1\%
    of editors. No significant difference in the other two groups.}

\end{frame}

\begin{frame}

  \frametitle{Do barnstars encourage editing? Results for Retention}

  \includegraphics[width=\textwidth]{figures/barnstars-2.pdf}

  \note{Lower retention among award recipients in the less active
    group! No significant difference in the other two groups.}
\end{frame}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%% SLIDE: Other Resources
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{More Resources}

  \begin{itemize}
    \larger \larger
  \item \e{Wikimedia Research Newsletter} [[:meta:Research:Newsletter]]
  \item \e{WikiSym} (Later this month in Berlin!)
  \item \e{WikiPapers Repository} [http://wikipapers.referata.com]
  \item \e{Much More}
  \end{itemize}

  {\centering 
    \includegraphics[width=0.25\textwidth]{figures/Wikimedia_Research_Newsletter_Logo.png}

  }

  \note{Those are my six exemplary studies from the past year. 

    There has been just tons and tons of work in this area. Trying to
    talk about this in 20 minutes strikes me as increasingly crazy
    every year I try to do it.

    The most important source, now going for a couple years, is the
    Wikimedia Research Newsletter which is published monthly in the (English)
    Signpost and syndicated on the Wikimedia Research.

    But there are other resources as well. And I encourage you to get
    involved.}

\end{frame}

\subsection{Meta-Analyses}

\begin{frame}

  \frametitle{Meta-Analyses}

  \begin{itemize}

    \larger \larger

  \item Okoli et al.,
    \href{https://spectrum.library.concordia.ca/978618/}{``The sum of
      all human knowledge'': a systematic review of scholarly research
      on the content of Wikipedia}.

  \item Bar-Ilan and Aharony,
    \href{http://dl.acm.org/citation.cfm?doid=2615569.2615643}{Twelve
      years of Wikipedia research}.
    
  \item Taraborelli. \href{https://meta.wikimedia.org/wiki/Research:Newsletter/2013/August\#Keynote\_on\_applicable\_Wikipedia\_research}{Keynote
      on Wikipedia Research}. OpenSym 2013. Hong Kong.

  \item Benkler, Shaw, and Hill,
    \href{http://mako.cc/academic/benkler\_shaw\_hill-peer\_production\_ci.pdf}{Peer
      Production: A Modality of Collective Intelligence}.

  \end{itemize}

\end{frame}


\end{document}

%  LocalWords:  xshift yshift makopurple Tilman Wikimedians Okoli al
%  LocalWords:  Ilan Aharony Taraborelli OpenSym Hong shaw ci pdf GFT
%  LocalWords:  McIver Brownstein Comput Wiktionary Acs Judit LREC Ou
%  LocalWords:  Rekyjavik Multi polysemy Nifrário Rodrigues Silvério
%  LocalWords:  Colaboração Massa Amadorismo Estudo Comparativo Da ou
%  LocalWords:  Qualidade Informação Científica Produzida Utilizando
%  LocalWords:  Conceitos Ferramentas Universidade Évora WikiProjects
%  LocalWords:  Weblogs Sethi Deepika DBpedia Restivo Arnout Rijt th
%  LocalWords:  WikiPapers