library(tidyverse)
library(cowplot)
library(ggthemes)
library(scales)

### LOAD DATA ###

# load SERPs
r_serps <- read_tsv('step8_serp_results.tsv', col_names = T) %>%
  mutate(relation_to_network = fct_relevel(relation_to_network, "own blog", "network/author mentioned", "not mentioned")) %>%
  mutate(search_type = fct_relevel(search_type, "plain", "verbatim", "googla")) %>%
  mutate(domain = Main %>% str_remove('^https?://(www.)?') %>% str_remove('/$'))

# load queries and combine source and role
r_queries <- read_tsv('step7_query_tracing.tsv', col_names = T) %>%
  mutate(date = as.Date(date)) %>%
  mutate(what=fct_cross(source, role)) %>%
  mutate(what=fct_recode(what, 'query in blog' = 'blog:googla',
                         'query in newspaper' = 'newspaper:googla',
                         'term in newspaper' = 'newspaper:general',
                         'term in blog' = 'blog:general',
                         'title in newspaper' = 'newspaper:title',
                         'title in blog' = 'blog:title'))

##### PLOT 1: timelines and SERPs of three different queries #####

# define a function for the timeline plot
# based on queries (D), serps (E)
# as well as which query_term to plot (qry), which timescale for binning to use (condense_by), and if legend should be printed
query_plot_timeline <- function(D, E, qry, condense_by="week", legend='none') {
  # filter query data
  D2 <- D %>% filter(query_term == qry) %>%
    group_by(what, mon = as.Date(cut(date, condense_by)), query_term) %>%
    summarise(n=n()) %>% ungroup()
  # define colour pallete
  local_pal <- tableau_color_pal('Color Blind')(8)
  # plot timeline
  ggplot(D2, aes(x=mon)) +
    geom_bar(aes(fill=what, y=n), stat='identity') +
    scale_x_date(breaks=breaks_pretty(4), date_labels='%b %Y') + theme_cowplot() + scale_y_continuous(breaks=breaks_pretty()) +
    scale_fill_manual(values=local_pal[c(6, 2, 1, 5, 4, 3)], limits=c('title in blog', 'title in newspaper', 'query in blog', 'query in newspaper', 'term in blog', 'term in newspaper')) +
    theme(plot.margin = unit(c(24, 0, 6, 0), "pt"), legend.box.margin = unit(c(0,0,0,10), 'pt')) +
    #theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) +
    theme(axis.text = element_text(size=10), axis.title = element_text(size=10)) +
    theme(legend.position=legend) + labs(x=paste0('date by ', condense_by), y = paste0('count'), fill='Type of occurrence')
}
# define a function for the ranking plot
# same inputs as for the above
query_plot_rank <- function(D, E, qry, condense_by="week", legend='none') {
  # filter query data
  E2 <- E %>% filter(query_term == qry) %>%
    # recode search engines for less text
    mutate(source=fct_recode(source, 'D'='duckduckgo', 'G' = 'google')) %>%
    # search type recoding
    mutate(search_type=fct_recode(search_type, 'imperative' = 'googla')) %>%
    mutate(search_type=fct_recode(search_type, 'simple' = 'plain')) %>%
    mutate(rank = as.factor(Position)) %>%
    # merge NA and not mentioned
    mutate(relation_to_network = replace_na(relation_to_network, 'not mentioned') %>% fct_recode('not mentioned / NA' = 'not mentioned'))
  # define colour pallette
  local_pal <- tableau_color_pal('Color Blind')(8)
  # plot data
  ggplot(E2, aes(x)) +
    annotate('text', label='no further\n  results', x='D', y=2.5, size=3,
             colour="grey60", lineheight=0.7, hjust=0.3) +
    geom_tile(aes(x=source, y=rank), fill='white', colour='black', linewidth=1, lineend='square') +
    geom_tile(aes(x=source, y=rank, fill=relation_to_network)) +
    scale_fill_manual(values=local_pal[1:3], na.value=c('white')) +
    scale_y_discrete(limits=rev, position='right') +
    facet_grid(. ~ search_type, switch = 'y') +
    scale_colour_grey() +
    theme_cowplot() + 
    theme(axis.text = element_text(size=10), axis.title = element_text(size=10), strip.text = element_text(size=10)) +
    theme(plot.margin = unit(c(28, 0, 0, 0), "pt"), legend.box.margin = unit(c(0,0,0,10), 'pt'), legend.key=element_rect(colour="black", linewidth = 1)) +
    theme(legend.position=legend) +
    labs(y='rank', x='search engine and query', fill = 'Related to the climate\nobstructionist group?')
}

# plot 3 each of timeline and rank plots
plot_grid(
  query_plot_timeline(r_queries, r_serps, 'som driver ipcc'),
  query_plot_rank(r_queries, r_serps, 'som driver ipcc'),
  query_plot_timeline(r_queries, r_serps, 'röda tråd nwo', 'quarter'),
  query_plot_rank(r_queries, r_serps, 'röda tråd nwo', 'quarter'),
  query_plot_timeline(r_queries, r_serps, 'koldioxidens verkan på klimatet', 'quarter'),
  query_plot_rank(r_queries, r_serps, 'koldioxidens verkan på klimatet', 'quarter'),
  # legend for timeline plot
  plot_grid(NULL,
            get_legend(query_plot_timeline(r_queries, r_serps, 'som driver ipcc', legend='right') + guides(fill=guide_legend(ncol=2, byrow=T)) + theme(legend.title = element_text(size=11), legend.text = element_text(size=9))),
            nrow=2, rel_widths = c(0.2, 0.8)), 
  # legend for rank plot
  plot_grid(NULL,
            get_legend(query_plot_rank(r_queries, r_serps, 'som driver ipcc', legend='right') + guides(fill=guide_legend(ncol=1)) + theme(legend.title = element_text(size=11), legend.text = element_text(size=9))),
            nrow=2, rel_widths = c(0.1, 0.8)),
  align = 'hv', axis='tblr', nrow = 4, ncol=2, label_size = 10,
  labels=c("(a.1) Timeline for keyphrase\n'what drives the_IPCC'",
           "(a.2) SERP for keyphrase\n'what drives the_IPCC'",
           "(b.1) Timeline for keyphrase\n'red thread nwo'",
           "(b.2) SERP for keyphrase\n'red thread nwo'",
           "(c.1) Timeline for keyphrase\n'carbon_dioxide’s effect on the_climate'",
           "(c.2) SERP for keyphrase\n'carbon_dioxide’s effect on the_climate' ",
           '',
           ''),
  hjust=0.5, label_x=0.6, vjust=1,
  rel_heights = c(1,1,1,0.4), rel_widths = c(1,0.7)
  # add subtitle
) %>% add_sub(label='Data based on the blog, searches in the Newspaper database Retriever\nand on SERPs from Google and DuckDuckGo.',
                       x=0.99, hjust=1, color='grey50', size = 10) %>%
  save_plot(filename='plot1_query_timelines.png',
                     base_height = unit(9, 'cm'), base_asp=0.9)


#### PLOT 2: overview of aggregated SERPs ####

# defining a function returning a cowplot grid based on data D, a subset of r_serps
# plot is a grid of barchart counting links on the left
# and boxplot with the ranking on the right
facet_panel <- function(D) {
# merge NA and not mentioned
D <- D %>% mutate(relation_to_network = replace_na(relation_to_network, 'not mentioned') %>% fct_recode('not mentioned / NA' = 'not mentioned'))
plot_grid( # barchart
  ggplot(D) +
    geom_bar(aes(x=relation_to_network, fill=relation_to_network), position = 'dodge', colour='black') +
    scale_fill_tableau(palette = 'Color Blind') + labs(y='count', x='') +
    scale_y_continuous(limits=c(0,220), breaks=seq(0,200,50)) +
    labs(caption ='number of results') +
    theme_cowplot() +
    theme(legend.position="none",
          axis.text.x = element_blank(), axis.ticks.x = element_blank(),
          axis.title.x = element_blank(),
          axis.text.y = element_text(size=10), axis.title.y = element_text(size=10)),
  # boxplot
  ggplot(D) +
    geom_boxplot(aes(y=Position, fill=relation_to_network), varwidth = T) +
    labs(y='rank', x='') + scale_fill_tableau(palette = 'Color Blind') + 
    scale_y_reverse(limits=c(10,1), breaks=seq(1, 10, 1), position='right') +
    labs(caption = 'rank in SERP') +
    theme_cowplot() +
    theme(legend.position="none",
          axis.text.x = element_blank(), axis.ticks.x = element_blank(),
          axis.title.x = element_blank(),
          axis.text.y = element_text(size=10), axis.title.y = element_text(size=10)),
  align = 'h', axis='bt', nrow = 1, rel_widths = c(1, 1.5)
) + theme(plot.margin = unit(c(24, 0, 0, 0), "pt"))
}

# plot the panel 6 times, once for each combination of search engine and search type
plot_grid(
  plot_grid(
    r_serps %>% filter(source == 'google', search_type == 'plain') %>% facet_panel,
    r_serps %>% filter(source == 'google', search_type == 'verbatim') %>% facet_panel,
    r_serps %>% filter(source == 'google', search_type == 'googla') %>% facet_panel,
    r_serps %>% filter(source == 'duckduckgo', search_type == 'plain') %>% facet_panel,
    r_serps %>% filter(source == 'duckduckgo', search_type == 'verbatim') %>% facet_panel,
    r_serps %>% filter(source == 'duckduckgo', search_type == 'googla') %>% facet_panel,
    align = 'hv', axis='tblr', nrow = 2, ncol=3, label_size = 12,
    labels=c('(a.1) Google, simple search\n[keyphrase]', '(b.1) Google, verbatim search\n\'[keyphrase]\'', '(c.2) Google, w/ imperative\ngoogla [keyphrase]',
             '(a.2) DuckDuckGo, simple search\n[keyphrase]', '(b.2) DuckDuckGo, verbatim search\n\'[keyphrase]\'', '(c.1) DuckDuckGo, w/ imperative\ngoogla [keyphrase]'),
    hjust=0.5, label_x=0.5, vjust=1),
  # add the legend on the bottom in its own grid for centering
  plot_grid(NULL, get_legend(
                      # merge NA and not mentioned
                      r_serps %>% mutate(relation_to_network = replace_na(relation_to_network, 'not mentioned') %>% fct_recode('not mentioned / NA' = 'not mentioned')) %>%
                      ggplot() + geom_boxplot(aes(y=Position, fill=relation_to_network)) +
                        scale_fill_tableau(palette = 'Color Blind') + theme_cowplot() +
                        theme(plot.margin = unit(c(-20, 0, 0, 0), "pt")) +
                        labs(fill = 'Related to the climate obstructionist group?') + theme(legend.position = 'bottom')),
            ncol=3, rel_widths = c(0.17, 1)),
  ncol=1, rel_heights = c(1, 0.01)
) %>% add_sub(label='Data based on SERPs related to 25 queries suggested in the climate obstructionists\' blog.',
                       x=0.99, hjust=1, color='grey50', size = 10) %>% 
save_plot(filename='plot2_boxplot_serps.png',
                   base_height = unit(8, 'cm'))


#### PLOT 3: coded domains and overlap of hyperlinks and SERPs ####

# load all hyperlinks and all coded domains
r_links <- read_tsv('step2_all_hyperlinks.tsv', col_names = T)
r_domains <- read_tsv('step10_coded_domains.tsv', col_names = T)

# overlap sersp and hyperlinks, keeping respective domain counts
overlap <- full_join(
  r_serps %>% filter(domain != 'klimatsans.com') %>%
    group_by(domain) %>%
    summarise(n = n(),
              relation_to_network_mean = mean(relation_to_network == "network/author mentioned", na.rm = T)),
  r_links %>% group_by(domain) %>% count(),
  by= join_by('domain'), multiple="all",
  suffix=c('.serps', '.links'),
) %>%
  # replace NaN with 0
  mutate(n.links = replace_na(n.links, 0), n.serps = replace_na(n.serps, 0)) %>%
  # merge with URL coding
  left_join(r_domains %>% select(domain, site_type, language), by = join_by('domain')) %>%
  # recoding of some variables for plotting
  # unify media site type
  mutate(site_type = if_else(site_type %in% c('media/sectoral', 'media/science', 'media/regional'), 'media', site_type)) %>%
  mutate(site_type = fct_lump_n(site_type, 2, other_level = "other /\nnot coded")) %>%
  mutate(site_type = replace_na(site_type, 'other /\nnot coded')) %>%
  # drop all but swedish language category
  mutate(language = fct_other(replace_na(language, 'x'), keep='swedish', other_level='other / not coded')) %>%
  # create three categories for share of mentions based on percentages
  mutate(mentions_sep = cut(replace_na(relation_to_network_mean, 0),
                            breaks=c(-0.01, 0.4999999, 1.01),
                            labels=c("<50% or no data", ">=50%"), ordered_result = T)) %>%
  arrange(-n.serps) %>%
  filter(n.links > 0, n.serps > 0)
  
# define a local colour pallete and how colours should be mapped
local_pal <- tableau_color_pal('Color Blind')(8)[c(3,2)]

# returns plot for SERPs or hyperlinks
# parameters data (D), plotted variable name for y axis is nx, 
plot.site_type <- function (D){
  ggplot(D %>% filter(nx > 0), aes(x=site_type)) +
  geom_boxplot(aes(y=nx, fill=mentions_sep), varwidth = T) +
  scale_fill_manual(values=local_pal) +
  theme_cowplot() + theme(axis.text = element_text(size=10), axis.title = element_text(size=10), strip.text = element_text(size=10)) +
  theme(plot.margin = unit(c(28, 0, 12, 0), "pt")) + scale_y_log10() +
  labs(x='', fill='share of results\nmentioning network/author', shape= 'site language', y='count (log)') +
  theme(legend.position = 'none', legend.text = element_text(size=8), legend.title = element_text(size=8, face = 'bold'))
}

# create overlap plot
plot.serplink <- ggplot(overlap %>% filter(n.links > 0, n.serps > 0), aes(x=n.serps, y=n.links)) +
  geom_smooth(method = "glm", se = F, colour='grey') +
  geom_jitter(aes(colour=mentions_sep), alpha=0.5, position=position_jitter(width=0.03, height=0.03, seed=10), size=2) +
  scale_colour_manual(values=local_pal) +
  scale_y_log10() + scale_x_log10() + theme_cowplot() +
  theme(axis.text = element_text(size=10), axis.title = element_text(size=10), strip.text = element_text(size=10)) +
  theme(plot.margin = unit(c(28, 0, 0, 0), "pt")) + theme(legend.text = element_text(size=8), legend.title = element_text(size=8, face = 'bold')) +
  labs(x='count of domain in all SERPs (log)', y='number of hyperlinks to domain in all blog posts (log)', colour='Network/authors\nmentioned in SERP results?')

# plot a grid of two items in the first row, one in the second row, legend in the third row
plot_grid(
  # row 1: serps and links
  plot_grid(
    overlap %>% rename(nx = n.serps) %>% plot.site_type() + labs(y='count of SERP results (log)'),
    overlap %>% rename(nx = n.links) %>% plot.site_type() + labs(y='count of hyperlinks (log)'),
    align='hv', axis='l', nrow = 1, label_size = 10,
    labels=c('(a) SERP domains by site type', '(b) Hyperlink domains by site type')),
  # row 2: connection
  plot.serplink + theme(legend.position = 'none'),
  # row 3: legend
  plot_grid(NULL,
            get_legend(overlap %>% rename(nx = n.serps) %>% plot.site_type() +
                         labs(colour='share of results mentioning\nnetwork/author (based on SERPs)') +
                         theme(legend.position = 'bottom') +
                         guides(colour=guide_legend(nrow=1))),
            rel_widths = c(0.2, 1)),
  rel_heights = c(1,1,0.2),
  ncol = 1, label_size = 10, hjust = c(-0.25, -0.5), axis = 'l',
  labels=c('',
           '(c) Domains occurring in both SERPs and hyperlinks')) %>%
  add_sub(label='Data based on the blog and SERPs from Google and DuckDuckGo.\nShowing only domains with at least related to at least one SERP and at least one hyperlink.',
                   x=0.99, hjust=1, color='grey50', size = 10) %>%
  save_plot(filename='plot3_overlap.png',
                     base_height = unit(8, 'cm'), base_asp = 0.9)