setwd('/Users/yyang/OneDrive - UC San Diego/China data lab/Sentiment Socre of Congressmen/shared_data/working_data/') load('./JulySubmission.Rdata') library(ggplot2) library(lubridate) library(dplyr) library(fpp2) library(zoo) library(yaml) social_list <- read_yaml("./bio_info/legislators-social-media.yaml") current <- read_yaml('./bio_info/legislators-current.yaml') socialList <-lapply(social_list,function(x) c(x$id$bioguide,x$social$twitter)) #complete observation missing ID socialList <- socialList[c(-316,-242,-211,-418,-437)] socialdf<- data.frame(matrix(unlist(socialList), nrow=length(socialList), byrow=T)) names(socialdf) <- c("Bioguide","Twitter_ID") socialdf$Twitter_ID_lower <- tolower(socialdf$Twitter_ID) bio_info <- lapply(current, function(x) c( x$id$bioguide, x$name$official_full, x$bio$birthday, x$bio$gender, tail(x$terms, n = 1) %>% lapply(function(x) c(x$type, x$start, x$end, x$state, x$party)) %>% unlist() )) %>% unlist() %>% matrix(ncol = 9, byrow = T) %>% data.frame() names(bio_info) <- c( 'ID', 'Name', 'Birth', 'Gender', 'Last_term_type', 'Last_term_start', 'Last_term_end', 'State', 'Party' ) covariateanaly <-JulySubmission %>% mutate_at(c('screen_name'),tolower)%>% left_join(socialdf, by=c('screen_name'='Twitter_ID_lower'))%>% left_join(bio_info,by=c('Bioguide'='ID')) Type_status <- c( rep = 'Representative', sen = 'Senator' ) blank_theme <- theme_minimal()+ theme( axis.title.x = element_blank(), axis.title.y = element_blank(), panel.border = element_blank(), panel.grid=element_blank(), axis.ticks = element_blank(), plot.title=element_text(size=14, face="bold") ) #Fig0 Backgroud figure # covariateanaly %>% # filter(created_at>as.Date('2016-01-01'))%>% # mutate(merged_party = case_when( # Party.x == 'Independent' ~ 'Democrat', # Party.x == 'Democrat' ~ 'Democrat', # Party.x == 'Republican' ~ 'Republican', # ))%>% # mutate(merged_party = factor(merged_party, levels = c('Democrat','Republican')), country = factor(country,levels = c('China','Iran','Canada')))%>% # ggplot(aes(x="",group=country, fill=country))+ # geom_bar(position = 'fill')+ # geom_text(aes(label=scales::percent(..count../sum(..count..))), # stat='count',position=position_fill(vjust=0.5))+ # xlab("")+ # ylab("")+ # labs(fill='Country',title='Congressional Members\' Tweets of Different Countries')+blank_theme+ # theme(axis.text.x=element_blank())+ # coord_polar('y',start=0) covariateanaly %>% filter(created_at>as.Date('2016-01-01'))%>% mutate(merged_party = case_when( Party.x.x == 'Independent' ~ 'Democrat', Party.x.x == 'Democrat' ~ 'Democrat', Party.x.x == 'Republican' ~ 'Republican', ))%>% mutate(merged_party = factor(merged_party, levels = c('Democrat','Republican')), country = factor(country,levels = c('China','Iran','Canada')))%>% ggplot(aes(x="",group=country, fill=country))+ geom_bar(position = 'fill')+ geom_text(aes(label=scales::percent(..count../sum(..count..))), stat='count',position=position_fill(vjust=0.5))+ xlab("")+ ylab("")+ labs(fill='Country',title='Congressional Members\' Tweets of Different Countries')+blank_theme+ theme(axis.text.x=element_blank())+ coord_polar('y',start=0) #Fig1 Party.x involvement toward different countries covariateanaly %>% filter(created_at>as.Date('2016-01-01') & country != 'Canada')%>% mutate(merged_party = case_when( Party.x.x == 'Independent' ~ 'Democrat', Party.x.x == 'Democrat' ~ 'Democrat', Party.x.x == 'Republican' ~ 'Republican', ))%>% mutate(merged_party = factor(merged_party, levels = c('Democrat','Republican')), country = factor(country,levels = c('China','Canada','Iran')))%>% group_by(Date = floor_date(created_at, unit = "month"),merged_party,country, Last_term_type.x)%>% summarise(n=n())%>% ggplot(aes(x=Date, y = n, fill=merged_party))+ #geom_line()+ #stat_smooth( method = lm,formula = y ~ poly(x, 3))+ #geom_area( position = 'stack')+ geom_bar(stat = 'identity',position = 'fill')+ facet_grid(country ~.)+ scale_y_continuous('Tweets frequency proportion',labels = scales::percent) + theme_classic()+ labs(fill = "Party.x",title='Party.x Involvement Toward Different Countries')+ scale_fill_manual(values = c(Democrat = "royalblue", Republican = "indianred")) #Fig3 Different sentiments toward different countries covariateanaly %>% filter(!is.na(AvgScore) & created_at > as.Date('2015-01-01'))%>% mutate(merged_party = case_when( Party.x.x == 'Independent' ~ 'Democrat', Party.x.x == 'Democrat' ~ 'Democrat', Party.x.x == 'Republican' ~ 'Republican', ))%>% mutate( country = factor(country,levels = c('China','Iran','Canada')))%>% #mutate(sentiment_ma07 = rollmean(avg_score, k = 7))%>% ggplot(aes(x= merged_party,y=AvgScore,fill=country))+ geom_jitter(aes(color=country,fill=country),alpha=0.5)+ geom_violin(scale="area",alpha=0.8)+ labs(title='AvgScore Difference Between Countries and Parties',x='Party.x',y='Average AvgScore Score')+ #geom_boxplot(alpha=0.8) scale_y_continuous(limits = c(1,5))+ theme_classic() ####Fig4 Topic Trend library(gridExtra) library(gtable) library(grid) grid_arrange_shared_legend <- function(..., nrow = length(list(...)), ncol = 1, position = c("top", "bottom")) { plots <- list(...) position <- match.arg(position) g <- ggplotGrob(plots[[1]] + theme(legend.position = position))$grobs legend <- g[[which(sapply(g, function(x) x$name) == "guide-box")]] lheight <- sum(legend$height) lwidth <- sum(legend$width) gl <- lapply(plots, function(x) x + theme(legend.position = "none")) gl <- c(gl, ncol = ncol, nrow = nrow) combined <- switch( position, "top" = arrangeGrob( do.call(arrangeGrob, gl), legend, ncol = 1, heights = unit.c(unit(1, "npc") - lheight, lheight) ), "bottom" = arrangeGrob( do.call(arrangeGrob, gl), legend, ncol = 2, widths = unit.c(unit(1, "npc") - lwidth, lwidth) ) ) grid.newpage() grid.draw(combined) # return gtable invisibly invisible(combined) } library(wesanderson) p1<-covariateanaly %>% filter(!is.na(hightoprec) & created_at > as.Date('2016-01-01')& (hightoprec=='V1' | hightoprec=='V2'|hightoprec=='V3'|hightoprec=='V4'))%>% mutate(merged_party = case_when( Party.x == 'Independent' ~ 'Democrat', Party.x == 'Democrat' ~ 'Democrat', Party.x == 'Republican' ~ 'Republican' ))%>% mutate(country = factor(country,levels = c('China','Iran','Canada')))%>% group_by(date = floor_date(created_at, unit = "month"),hightoprec)%>% summarise(n=n())%>% ggplot(aes(x=date,y=n,fill=hightoprec))+ geom_area()+ labs(fill='Topics',y='Tweets Number',x='Date',title = 'Congressional Member Tweets Topic Timetrend')+ scale_fill_manual(values=wes_palette(n=4, name="Darjeeling1"),name = "Topics", labels = c( "National Security", "Human Rights/Advocacy", "Trade", "Global Governance"))+ theme(plot.title = element_text(size=12,hjust=0.5))+ theme(legend.position="bottom") p2<-covariateanaly %>% filter(!is.na(hightoprec) & created_at > as.Date('2016-01-01')& (hightoprec=='V2' | hightoprec=='V3'|hightoprec=='V4'|hightoprec=='V6'))%>% mutate(merged_party = case_when( Party.x == 'Independent' ~ 'Democrat', Party.x == 'Democrat' ~ 'Democrat', Party.x == 'Republican' ~ 'Republican' ))%>% mutate(country = factor(country,levels = c('China','Iran','Canada')))%>% group_by(date = floor_date(created_at, unit = "month"),hightoprec)%>% summarise(n=n())%>% ggplot(aes(x=date,y=n,fill=hightoprec))+ geom_bar(stat = 'identity',position = 'fill')+ labs(fill='Topics',y='Topic Proportion',x='Date',title='Topic Proportion Time Trend')+ scale_fill_manual(values=wes_palette(n=4, name="Darjeeling1"),name = "Topics", labels = c( "National Security", "Human Rights/Advocacy", "Trade", "Global Governance"))+ theme(legend.position="bottom")+ theme(plot.title = element_text(size=12,hjust=0.5)) grid_arrange_shared_legend(p1, p2) ####Fig4_2 Time Trend of Congressional Twitter Sentiment by Country covariateanaly %>% filter(!is.na(AvgScore) & created_at > as.Date('2015-01-01'))%>% mutate(merged_party = case_when( Party.x == 'Independent' ~ 'Democrat', Party.x == 'Democrat' ~ 'Democrat', Party.x == 'Republican' ~ 'Republican', ),country = factor(country,levels = c('China','Iran','Canada')))%>% group_by(date = floor_date(created_at, unit = "month"),country)%>% summarise(avgscore=mean(AvgScore))%>% ggplot(aes(x=date,y=avgscore,group=country,col=country))+ geom_point()+ geom_smooth()+ labs(col='Country',y='Sentiment Score',x='Date',title = 'Time Trend of Congressional Twitter Sentiment by Country')+ theme(plot.title = element_text(size=12,hjust=0.5))+ scale_y_continuous(labels = scales::number_format(accuracy = 0.1)) ####Fig4_3 Time Trend of Congressional Sentiment Towards China by Partisanship covariateanaly %>% filter(!is.na(AvgScore) & created_at > as.Date('2015-01-01') & country=='China')%>% mutate(merged_party = case_when( Party.x == 'Independent' ~ 'Democrat', Party.x == 'Democrat' ~ 'Democrat', Party.x == 'Republican' ~ 'Republican', ))%>% group_by(date = floor_date(created_at, unit = "month"),merged_party)%>% summarise(avgscore=mean(AvgScore))%>% ggplot(aes(x=date,y=avgscore,group=merged_party,col=merged_party))+ geom_point()+ geom_smooth()+ labs(col='Party',y='Sentiment Score',x='Date',title = 'Time Trend of Congressional Sentiment Towards China by Partisanship')+ scale_color_manual(values = c(Democrat = "royalblue", Republican = "indianred"))+ theme(plot.title = element_text(size=12,hjust=0.5))+ scale_y_continuous(labels = scales::number_format(accuracy = 0.1)) #Fig5 Topics between countries covariateanaly %>% filter(!is.na(hightoprec) & created_at > as.Date('2016-01-01')& (hightoprec=='V1' | hightoprec=='V2'|hightoprec=='V3'|hightoprec=='V4'))%>% mutate(merged_party = case_when( Party.x == 'Independent' ~ 'Democrat', Party.x == 'Democrat' ~ 'Democrat', Party.x == 'Republican' ~ 'Republican', ))%>% mutate(country = factor(country,levels = c('China','Iran','Canada')))%>% group_by(hightoprec,country)%>% summarise(n=n())%>% ggplot(aes(x=hightoprec,y=n,fill=country))+ geom_bar(stat = 'identity',position='dodge')+ labs(x='Topics',y='Tweets Number', fill='Country')+ coord_flip()+ scale_fill_discrete(name = "Country", labels = c( "China", "Iran", "Canada"))+ labs(x='Topics',title='Topic Distribution')+ scale_x_discrete(labels=c( 'V1' = "National Security", "V2" = "Human Rights/Advocacy", "V3" = "Trade", 'V4' = "Global governance"))+ theme(legend.position="bottom") #Fig6 Topic sentitment score between parties covariateanaly %>% filter(!is.na(AvgScore)&!is.na(hightoprec) & created_at > as.Date('2016-01-01')& (hightoprec=='V1' | hightoprec=='V2'|hightoprec=='V3'|hightoprec=='V4'))%>% mutate(merged_party = case_when( Party.x == 'Independent' ~ 'Democrat', Party.x == 'Democrat' ~ 'Democrat', Party.x == 'Republican' ~ 'Republican', ))%>% mutate(country = factor(country,levels = c('China','Iran','Canada')))%>% group_by(hightoprec,country,merged_party)%>% summarise(AvgScore=mean(AvgScore))%>% ggplot(aes(x=country,y=AvgScore,group=merged_party,fill=merged_party))+ geom_bar(stat = 'identity', position = 'dodge')+ facet_grid(hightoprec~.,labeller = labeller(hightoprec = c( 'V1' = "National Security", "V2" = "Human Rights/Advocacy", "V3" = "Trade", 'V4' = "Global governance" )))+ labs(fill='Party.x',x='Country',y='Average AvgScore Socre',title='Topic Sentitment Somparison Between Parties')+ scale_fill_manual(values = c(Democrat = "royalblue", Republican = "indianred")) #Fig7 AvgScore Score between topics covariateanaly %>% filter(!is.na(AvgScore)&!is.na(hightoprec) & created_at > as.Date('2016-01-01')& (hightoprec=='V1' | hightoprec=='V2'|hightoprec=='V3'|hightoprec=='V4'))%>% mutate(merged_party = case_when( Party.x == 'Independent' ~ 'Democrat', Party.x == 'Democrat' ~ 'Democrat', Party.x == 'Republican' ~ 'Republican', ))%>% mutate(country = factor(country,levels = c('China','Iran','Canada')))%>% group_by(hightoprec,country,merged_party)%>% summarise(AvgScore=mean(AvgScore))%>% ggplot(aes(x=hightoprec,y=AvgScore,group=country,fill=country))+ geom_bar(stat = 'identity', position = 'dodge')+ facet_grid(merged_party~.)+ scale_x_discrete(labels=c( 'V1' = "National Security", "V2" = "Human Rights/Advocacy", "V3" = "Trade", 'V4' = "Global governance"))+ labs(fill='Party.x',x='Country',y='Average AvgScore Socre',title='AvgScore comparison Between Topics') #Fig8 China topics covariateanaly %>% filter(!is.na(AvgScore)&!is.na(hightoprec) & country == 'China' &created_at > as.Date('2016-01-01')& (hightoprec=='V1' | hightoprec=='V2'|hightoprec=='V3'|hightoprec=='V4'))%>% mutate(merged_party = case_when( Party.x == 'Independent' ~ 'Democrat', Party.x == 'Democrat' ~ 'Democrat', Party.x == 'Republican' ~ 'Republican', ))%>% #bind_rows(hightech)%>% group_by(hightoprec,country,merged_party)%>% summarise(AvgScore=mean(AvgScore))%>% ggplot(aes(x=hightoprec,y=AvgScore,group=merged_party,fill=merged_party))+ geom_bar(stat = 'identity', position = 'dodge')+ scale_x_discrete(labels=c( 'V1' = "National Security", "V2" = "Human Rights/Advocacy", "V3" = "Trade", 'V4' = "Global governance" ))+ labs(fill='Party',x='Topic',y='Average AvgScore Socre',title='AvgScore comparison Between Parties - China')+ scale_fill_manual(values = c(Democrat = "royalblue", Republican = "indianred")) covariateanaly %>% filter(!is.na(AvgScore) & created_at > as.Date('2015-01-01'))%>% mutate(Party.x = factor(Party.x, levels = c('Independent','Democrat','Republican')), country = factor(country,levels = c('China','Canada','Iran')))%>% group_by(date = floor_date(created_at, unit = "day"),Party.x,country = country)%>% summarise(n=n(),avg_score = mean(AvgScore))%>% #mutate(sentiment_ma07 = rollmean(avg_score, k = 7))%>% ggplot(aes(x= date,y=avg_score,group=Party.x,color=Party.x))+ # geom_line()+ geom_line(aes(y=rollmean(avg_score,7,na.pad=TRUE)))+ #stat_smooth( method = lm,formula = y ~ poly(x, 3))+ scale_y_continuous(limits = c(1,5))+ facet_grid(country ~ .)+ theme_classic()+ scale_color_manual(values = c(Democrat = "royalblue", Republican = "indianred", Independent = 'lightgray')) covariateanaly ggplot(covariateanaly,aes(created_at,y=AvgScore,group=country))+ geom_bar()