library(stringr) library(stm) library(jsonlite) library(wordcloud2) setwd("/Users/yyang/Dropbox/China data lab/viz/On censorship/share code/") #Load datasets change weibo <- read.csv("result_weibo_segmented.csv",encoding='UTF-8') twitter <- read.csv("result_twitter_segmented_random6000.csv",encoding='UTF-8') weibo$weibo<-c(1) twitter$weibo<-c(0) names(weibo)[names(weibo)=="weibo_text"]="text"; zh<- rbind(weibo, twitter) zh$text <- gsub("[_a-zA-Z0-9\u4e00-\u9fa5]+@|@\\s*[_a-zA-Z0-9\u4e00-\u9fa5]+", "", zh$text) #remove at zh$text <- gsub('\\p{So}|\\p{Cn}', '', zh$text, perl = TRUE)#remove emoji #zh$text[876] zh$text = gsub("\\[.*\\]", "", zh$text)#remove weibo emoji zh$text = gsub("&", "", zh$text)#remove & zh$text = gsub("(RT|via)((?:\\b\\W*@\\w+)+)", "", zh$text) #remove retweet #zh$text = gsub("[[:punct:]]", "", zh$text) #remove punctuation zh$text = gsub("http\\w+", "", zh$text)#remove url zh$text = gsub("[ \t]{2,}", "", zh$text)#remove zh$text = gsub("^\\s+|\\s+$", "", zh$text) #baidu_stopwords<-read.delim("stopwords/baidu.txt") json_stopwords <- fromJSON("stopwords/stopwords-zh.json") #Create a term document matrix for the mainland sample processed <- textProcessor(zh$text, zh, wordLengths=c(2,Inf),lowercase=F,stem=F,customstopwords = json_stopwords) out <- prepDocuments(processed$documents, processed$vocab,processed$meta) #### ##draw Wordcloud ### #Create term doc matrix doc.to.tdm <- function(documents, vocab){ tdm <- matrix(0,nrow=length(documents), ncol=length(vocab)) for(i in 1:length(documents)){ tdm[i,documents[[i]][1,]] <- documents[[i]][2,] } return(tdm) } tdm <- doc.to.tdm(out$documents, out$vocab) #Remove stopwords colnames(tdm) <- out$vocab #tdm <- tdm[,!out$vocab%in%stopwords] #Top words weibotdm <- tdm[out$meta$weibo==1,] weibotopwords <- apply(weibotdm,2,sum) #weibozhtopwords <- colnames(tdm)[order(weibotopwords,decreasing=T)[1:50]] twittertdm <- tdm[out$meta$weibo==0,] twittertopwords <- apply(twittertdm,2,sum) #Preparing data frame for drawing weibotopwords_freq <-data.frame(text=enc2utf8(as.character(colnames(tdm)[order(weibotopwords,decreasing=T)[1:500]])), value=order(weibotopwords,decreasing=T)[1:500],#value = ceiling(order(weibotopwords,decreasing=T)[1:1000]/100), group='weibo') twittertopwords_freq <-data.frame(text=enc2utf8(as.character(colnames(tdm)[order(twittertopwords,decreasing=T)[1:500]])), value=order(twittertopwords,decreasing=T)[1:500],#value = ceiling(order(twittertopwords,decreasing=T)[1:1000]/100), group='twitter') #export the json #export<- rbind(weibotopwords_freq, twittertopwords_freq) #word_json<-toJSON(export,pretty=TRUE) #cat(word_json, file = 'word_data.json', fill = FALSE, append = FALSE) #Draw wordcloud weibofigPath = "weibo-website-logo.png" twitterfigPath = "twitter_shape.png" wordcloud2(weibotopwords_freq, gridSize=4,figPath=weibofigPath,color = "red") wordcloud2(twittertopwords_freq,figPath=twitterfigPath,color = "skyblue") #Topics analysis ###Interlude to run an STM for fun #stm.out <- stm(out$documents, out$vocab, K=50, # prevalence = ~weibo, data=out$meta, init.type="Spectral") #save(stm.out, file="WeiboTwitterSTMJustPrevalence.RData") load("WeiboTwitterSTMJustPrevalence.RData") labels<-labelTopics(stm.out) #Political topics findThoughts(stm.out, out$meta$text, c(40,20,8), n=10) #prep <- estimateEffect(c(1:50) ~ weibo, stm.out, out$meta) #ests <- unlist(plot(prep, "weibo", method="difference", cov.value1 = 1, cov.value2 = 0)$means) #orderests <- order(ests) #Alternatively weiboprop <- apply(stm.out$theta[out$meta$weibo==1,], 2, mean) twitterprop <- apply(stm.out$theta[out$meta$weibo==0,], 2, mean) ests <- weiboprop - twitterprop orderests <- order(ests) #Political topics: 40,20 #40 Presidential election #20 Financial reports twittertopics_label<-labels$prob[orderests[1:5],] twittertopics <- c(orderests[1:5]) findThoughts(stm.out, out$meta$text, twittertopics, n=15) #findThoughts(stm.out, out$meta$text, twittertopics, topics=5) #29 Travel in China #22 English tweets #46 English tweets with political opinions #30 Block chain #8 Taiwan issue weibotopics <-c(orderests[46:50]) weibotopics_label<-labels$prob[orderests[46:50],] findThoughts(stm.out, out$meta$text, weibotopics, n=15) findThoughts(stm.out, out$meta$text, 20, n=30) #42: Taobao and Tmall goods #21: Constellation #14: Cooking and receipt #2: Marketing and promotions #9: Happy birthday to a celebrity concluded_labels <- c("Travel in China", "English tweets", "English tweets with political opinions", "Block chain", "Taiwan issue", "Chinese People's Congress and Presidential Election","Financial resports","Taobao and Tmall goods", "Constellation topics", "Cooking and receipt","Marketing and promotions","Happy birthday to a celebrity") topic_export<-data.frame(labels=concluded_labels, keywords=c( paste(labels$prob[orderests[1:5],1],labels$prob[orderests[1:5],2],labels$prob[orderests[1:5],3],labels$prob[orderests[1:5],4],labels$prob[orderests[1:5],5],labels$prob[orderests[1:5],6],labels$prob[orderests[1:5],7],sep=","), paste(labels$prob[40,1],labels$prob[40,2],labels$prob[40,3],labels$prob[40,4],labels$prob[40,5],labels$prob[40,6],labels$prob[40,7],sep=","), paste(labels$prob[20,1],labels$prob[20,2],labels$prob[20,3],labels$prob[20,4],labels$prob[20,5],labels$prob[20,6],labels$prob[20,7],sep=","), paste(labels$prob[orderests[46:50],1],labels$prob[orderests[46:50],2],labels$prob[orderests[46:50],3],labels$prob[orderests[46:50],4],labels$prob[orderests[46:50],5],labels$prob[orderests[46:50],6],labels$prob[orderests[46:50],7],sep=",")), weibo=c(weiboprop[orderests[1:5]],weiboprop[40],weiboprop[20],weiboprop[orderests[46:50]]), twitter=c(twitterprop[orderests[1:5]],twitterprop[40],twitterprop[20],twitterprop[orderests[46:50]])) #topic_export<-data.frame(labels=labels,difference=c(ests[orderests[1:5]],ests[40],ests[20],ests[orderests[46:50]])) topic_json<-toJSON(topic_export,pretty=TRUE) #cat(topic_json, file = 'topic_data.json', fill = FALSE, append = FALSE)