source("R/SetPackages.R")
source("R/Data.R")
source("R/Visualize.R")
source("R/MapsSpain.R")

  #.rs.restartR()

  ########## INITIALIZE ##########
  caesp = initMap()
  txt_df <- readRDS("otherfiles/localizaciones.rds") #documento con todas las localizaciones
  RDS_dir <- "databdd"
  coord <- c("40.36329,-3.69141,750km", "28.469648,-16.2540884,300km") #radio acotado de españa y de las islas canarias
  path_out_robos = "databddout/all_robos_unique.rds"
  path_out_seguridad = "databddout/all_seguridad_unique.rds"
  # Inicializamos la bdd de robos
  initBDD(RDS_dir, coord, tags = c("#Robo","Robos"), pattern_file_name = "_robos_esp", path_out_robos)
    
  # Inicializamos la bdd de seguridad
  initBDD(RDS_dir, coord, tags = c("#Seguridad", "#Proteccion","#Delincuencia", "delitos"), pattern_file_name = "_seguridad_esp", path_out_seguridad)
  
  ########## UNIGRAM ##########
  
  #inicializamos el corpus
  rt_robos = getAtributesRT(path_out_robos, txt_df)
  rt_seguridad = getAtributesRT(path_out_seguridad, txt_df)
  corp <- quanteda::corpus(rbind(rt_robos, rt_seguridad))
    
  #clasificamos cada tópico con un nombre. Encontrando tres: Robo, Delincuencia y Seguridad
  corp$clasify <- factor(ifelse(corp$query == "#Robo" | corp$query == "Robos", "Robo", ifelse(corp$query == "#Seguridad" | corp$query == "#Proteccion", "Seguridad","Delincuencia")))
  
  #creamos tokens del corpus
  toks <- clean_and_generate_tokens(corp)
  
  #definimos la matriz dfm.
  dfm_toks <- clean_dfm(toks)
  
  #definimos los ficheros que son de cada tópico
  isrobos <- docvars(corp)$clasify == "Robo"
  isseguridad <- docvars(corp)$clasify == "Seguridad"
  isdelincuencia <- docvars(corp)$clasify == "Delincuencia"
  
  visualizeLocationUsers(docvars(corp)[isrobos, ]) #Localización de usuarios en las diversas zonas. 
  visualizeLocationUsers(docvars(corp)[isseguridad, ]) #Localización de usuarios en las diversas zonas. 
  visualizeLocationUsers(docvars(corp)[isdelincuencia, ]) #Localización de usuarios en las diversas zonas. 
  
  
  #Visualización del número de tweets que se han hecho cada día
  visualizeNumTweetsbyDay(docvars(corp), docvars(corp)$created_at, docvars(corp)$clasify)
  
  #comparativa de los tres grupos en densidad.
  visualizeDensityTweets(docvars(corp), docvars(corp)$created_at, isseguridad, isrobos, isdelincuencia)
  
  #nube de palabras dividida por los tres grupos
  visualizewordcloud(dfm_toks, docvars(corp)$clasify, max_size = 4)
  
  #conteo de casos en la bdd.
  print(docvars(corp) %>% count(lang, sort=TRUE))
  
  #conteo de casos teniendo en cuenta que place_name no esté vacío
  print(docvars(corp) %>% filter(place_name != "", !is.na(place_name)) %>% count(lang, sort=TRUE))
  
  #cluster dendrogram 
  cluster_dendrogram(dfm_toks, docvars(corp)$clasify)
  #cluster_dendrogram(dfm_toks, docvars(corp)$screen_name)
  
  # frecuencias de palabras
  visualizeFreqWords(dfm_toks, isrobos, isseguridad, isdelincuencia, maxfeatures=20)
    
  #hacemos comparativa de los tweets clasificados como Seguridad contra todo lo demás.
  visualizeKeyness(getKeyness(dfm_toks, isseguridad))
  
  #definimos el diccionario de sentimientos positivos y negativos.
  txt_pos <- readtext::readtext("otherfiles/positive_words_es.txt")
  txt_pos <- strsplit(txt_pos$text, "\n")
  txt_neg <- readtext::readtext("otherfiles/negative_words_es.txt")
  txt_neg <- strsplit(txt_neg$text, "\n")
  
  dict <- dictionary(list(positive = txt_pos, negative = txt_neg)) 
  quanteda.sentiment::polarity(dict) <- list(pos = "positive", neg = "negative")
  
  sentimentharvest <- getSentimentPolarity(dfm_toks, dict, fun=sent_logit, docvars(corp)) 
  #default log(pos/neg)
  
  sentimentharvest_abspropdiff <- getSentimentPolarity(dfm_toks, dict, fun=sent_abspropdiff, docvars(corp)) 
  #(pos-neg)/total  

  sentimentharvest_relpropdiff <- getSentimentPolarity(dfm_toks, dict, fun=sent_relpropdiff, docvars(corp))   
  #(pos - neg)/(pos+neg)
  
  #sentiment comparativa logic, abspropdiff y relpropdiff
  g1 = visualizeSentiment(sentimentharvest, sentimentharvest$created_at, sentimentharvest$sentiment, sentimentharvest$clasify,"Date of created","Sentiment")
  g2 = visualizeSentiment(sentimentharvest_abspropdiff, sentimentharvest_abspropdiff$created_at, sentimentharvest_abspropdiff$sentiment, sentimentharvest_abspropdiff$clasify,"Date of created","Sentiment Abspropdiff")
  g3 = visualizeSentiment(sentimentharvest_relpropdiff, sentimentharvest_relpropdiff$created_at, sentimentharvest_relpropdiff$sentiment, sentimentharvest_relpropdiff$clasify,"Date of created","Sentiment Relpropdiff")
  gridExtra::grid.arrange(g1, g2, g3, nrow = 3)
  
  grupsentiment <- meanSentiment(sentimentharvest)
  grupsentiment_abspropdiff <- meanSentiment(sentimentharvest_abspropdiff)
  grupsentiment_relpropdiff <- meanSentiment(sentimentharvest_relpropdiff)
  
  #mean del sentiment comparativa logic, abspropdiff y relpropdiff  
  g1 = visualizeSentiment(grupsentiment, grupsentiment$created_at, grupsentiment$mean_sentiment, grupsentiment$clasify,"Date of created","Sentiment Logic")
  g2 = visualizeSentiment(grupsentiment_abspropdiff, grupsentiment_abspropdiff$created_at, grupsentiment_abspropdiff$mean_sentiment, grupsentiment_abspropdiff$clasify,"Date of created","Sentiment Abspropdiff")
  g3 = visualizeSentiment(grupsentiment_relpropdiff, grupsentiment_relpropdiff$created_at, grupsentiment_relpropdiff$mean_sentiment, grupsentiment_relpropdiff$clasify,"Date of created","Sentiment Relpropdiff")
  
  gridExtra::grid.arrange(g1, g2, g3, nrow = 3)

  #hacemos la media del sentimiento por cada uno de los días que tenemos registrados en la
  #variable create_at, para cada una de las clasificaciones hechas.
  
  g1 = visualizeSentimentPolarity(sentimentharvest, "Sentiment Analysis Twitter in Spain", "Data Logic Spain")
  g2 = visualizeSentimentPolarity(sentimentharvest_abspropdiff, "Sentiment Analysis Twitter in Spain", "Data Abspropdiff Spain")
  g3 = visualizeSentimentPolarity(sentimentharvest_relpropdiff, "Sentiment Analysis Twitter in Spain", "Data Relpropdiff Spain")

  gridExtra::grid.arrange(g1, g2, g3, nrow = 3)
  
  sentimentlocations <- sentimentharvest %>% 
    select(location) %>%
    unique() %>%
    pull()
  
  coords <- sentimentlocations %>% opencage::oc_forward_df() 
  names(coords)[names(coords) == "placename"] <- "location"
  sentimentharvest_coords <- sentimentharvest %>% inner_join(coords, by = "location")

  sentiment_ccaa <- getSentimentCCAA(sentimentharvest_coords, txt_df)
    
  meanmap <- getMeanMap(sentiment_ccaa)
  
  visualizeSentimentMap(meanmap[meanmap$clasify == "Robo",], meanmap[meanmap$clasify == "Robo",]$sentiment, "Sentiment")
  visualizeSentimentMap(meanmap[meanmap$clasify == "Seguridad",],meanmap[meanmap$clasify == "Seguridad",]$sentiment, "Sentiment")
  visualizeSentimentMap(meanmap[meanmap$clasify == "Delincuencia",],meanmap[meanmap$clasify == "Delincuencia",]$sentiment, "Sentiment")
  
  visualizeSentimentMap(meanmap[meanmap$clasify == "Robo",], meanmap[meanmap$clasify == "Robo",]$total, "Total")
  visualizeSentimentMap(meanmap[meanmap$clasify == "Seguridad",],meanmap[meanmap$clasify == "Seguridad",]$total, "Total")
  visualizeSentimentMap(meanmap[meanmap$clasify == "Delincuencia",],meanmap[meanmap$clasify == "Delincuencia",]$total, "Total")

  #m <- getMap(sentiment_ccaa)
  #visualizePolygonMap(m)
  
  ########## NGRAMS2 ##########

  #inicializamos el corpus
  rt_robos = getAtributesRT(path_out_robos, txt_df)
  rt_seguridad = getAtributesRT(path_out_seguridad, txt_df)
  corp <- quanteda::corpus(rbind(rt_robos, rt_seguridad))
  
  
  #clasificamos cada tópico con un nombre. Encontrando tres: Robo, Delincuencia y Seguridad
  corp$clasify <- factor(ifelse(corp$query == "#Robo" | corp$query == "Robos", "Robo", ifelse(corp$query == "#Seguridad" | corp$query == "#Proteccion", "Seguridad","Delincuencia")))
  
  #definimos los ficheros que son de cada tópico
  isrobos <- docvars(corp)$clasify == "Robo"
  isseguridad <- docvars(corp)$clasify == "Seguridad"
  isdelincuencia <- docvars(corp)$clasify == "Delincuencia"
  
  #creamos tokens del corpus
  toks <- clean_and_generate_tokens(corp)
  
  #definimos la matriz de ngrams2
  ngrams2_toks <- toks %>%
    tokens_ngrams(n = 2, concatenator = " ") 
  
  dfm_ngrams2_toks <- clean_dfm(ngrams2_toks)
  
  #nube de palabras de ngrams2 dividida por los tres grupos
  visualizewordcloud(dfm_ngrams2_toks, docvars(corp)$clasify, max_size = 1.8)
  
  #cluster dendrogram ngrams2
  #cluster_dendrogram(dfm_ngrams2_toks, docvars(corp)$clasify)
  
  # frecuencias de palabras ngrams2
  visualizeFreqWords(dfm_ngrams2_toks, isrobos, isseguridad, isdelincuencia, maxfeatures=15)
  
  #hacemos comparativa de los tweets clasificados como Seguridad contra todo lo demás.
  visualizeKeyness(getKeyness(dfm_ngrams2_toks, isseguridad))
  
  ng <- textcnt(ngrams2_toks, method = "string", n=2)
  ngram <- data.table(w1w2 = names(ng), freq = unclass(ng), length=nchar(names(ng)))
  bigram <- ngram %>% separate(w1w2,c( "word1", "word2"), sep = " ") %>% 
    mutate(sentiment = quanteda.sentiment::textstat_polarity(word2, dictionary = dict)$sentiment + ifelse((word1 != "no" | word1 != "nunca"), quanteda.sentiment::textstat_polarity(word1, dictionary = dict)$sentiment, 0.0)) %>%
    mutate(sentiment = ifelse((word1 == "no" | word1 == "nunca") & sentiment != 0.0, -sentiment, sentiment)) %>% 
    mutate(ngram = paste(word1, word2))
  
  dictbigram <- bigram %>% select(ngram, sentiment)
  names(dictbigram)[names(dictbigram) == "ngram"] <- "word"
  dictbigram <- as.dictionary(dictbigram)
  
  sentiment_idx  <- ngrams2_toks %>% tokens_lookup(dictionary = dictbigram)
  sentiment_idx  <- lapply(sentiment_idx, as.double)
  sentiment_idx  <- lapply(sentiment_idx, mean)
  
  sentiment_toks <- data.frame("doc_id" = names(sentiment_idx), "sentiment" = as.double(as.character(sentiment_idx))) %>%  
    mutate(sentiment = replace_na(sentiment, 0.0)) 

  sentimentharvest <- getPolarity(sentiment_toks, docvars(corp))
  grupsentiment <- meanSentiment(sentimentharvest)
  
  visualizeSentiment(sentimentharvest, format(sentimentharvest$created_at, "%m %d"), sentimentharvest$sentiment, sentimentharvest$clasify,"Date of created","Sentiment Logic")
  visualizeSentiment(grupsentiment, grupsentiment$created_at, grupsentiment$mean_sentiment, grupsentiment$clasify, "Date of created", "Sentiment Logic")
  visualizeSentimentPolarity(sentimentharvest, "Sentiment Analysis Twitter in Spain", "Data Logic Spain")

  sentiment_by_country <- sentimentharvest %>% 
    group_by(location, clasify) %>% 
    summarise(country_sentiment = sum(sentiment)/n())
  
  sentimentlocations <- sentimentharvest %>% 
    select(location) %>%
    unique() %>%
    pull()
  
  coords <- sentimentlocations %>% opencage::oc_forward_df()
  names(coords)[names(coords) == "placename"] <- "location"
  sentimentharvest_coords <- sentimentharvest %>% inner_join(coords, by = "location")
  
  sentiment_ccaa <- getSentimentCCAA(sentimentharvest_coords, txt_df)
  
  meanmap <- getMeanMap(sentiment_ccaa)
  
  visualizeSentimentMap(meanmap[meanmap$clasify == "Robo",], meanmap[meanmap$clasify == "Robo",]$sentiment, "Sentiment")
  visualizeSentimentMap(meanmap[meanmap$clasify == "Seguridad",],meanmap[meanmap$clasify == "Seguridad",]$sentiment, "Sentiment")
  visualizeSentimentMap(meanmap[meanmap$clasify == "Delincuencia",],meanmap[meanmap$clasify == "Delincuencia",]$sentiment, "Sentiment")
  
  visualizeSentimentMap(meanmap[meanmap$clasify == "Robo",], meanmap[meanmap$clasify == "Robo",]$total, "Total")
  visualizeSentimentMap(meanmap[meanmap$clasify == "Seguridad",],meanmap[meanmap$clasify == "Seguridad",]$total, "Total")
  visualizeSentimentMap(meanmap[meanmap$clasify == "Delincuencia",],meanmap[meanmap$clasify == "Delincuencia",]$total, "Total")