### Load needed package
library(tidyverse)
library(lubridate)
library(rvest)
library(httr)

### Choose keywords needed then search on Dcard
url_raw <- "https://www.dcard.tw/search/general?query=zara"

### using rvest::read_html() to parse the raw url
html_raw <- url_raw %>% read_html()

### Use rvest:html_nodes() to parse different nodes of raw html
index_title <- html_raw %>% html_nodes(".PostEntry_unread_2U217-") %>% html_text()
index_link <- html_raw %>% html_nodes(".PostEntry_root_V6g0rd") %>% html_attr("href")
index_board <- html_raw %>% html_nodes(".PostEntry_forum_1m8nJA") %>% html_text()
index_date <- html_raw %>% html_nodes(".PostEntry_published_229om7") %>% html_text()
index_author <- html_raw %>% html_nodes(".hSAyoj") %>% html_nodes(".PostAuthor_root_3vAJfe") %>% html_text()
index_meta <- html_raw %>% html_nodes(".PostEntry_meta_1lGUFm") %>% html_text()
index_content <- html_raw %>% html_nodes(".PostEntry_content_g2afgv") %>% html_text()

### Bind columns then turn them to a dataframe 
df_index <- tibble(index_date,index_title,index_link,index_board,index_author,index_meta,index_content) %>%
  mutate(index_link = str_c("https://www.dcard.tw", index_link)) %>%
  mutate(index_title = str_remove_all(index_title,"\\(|\\)")) %>%
  mutate(index_title = str_remove_all(index_title,"\\（|\\）")) %>%
  mutate(index_title = str_remove_all(index_title,"\\？|\\?")) %>%
  mutate(index_title = str_remove_all(index_title,"\\[|\\〕")) %>%
  mutate(index_content = str_remove(index_content, index_title)) %>%
  mutate(index_content = str_remove(index_content, index_meta)) %>%
  mutate(index_content = str_remove_all(index_content, "\\\n")) %>%
  rename(index_excerpt = index_content) %>%
  mutate(index_date = str_replace_all(index_date, "月", "-")) %>%
  mutate(index_date = str_remove_all(index_date, "日")) %>%
  mutate(index_date = str_c(lubridate::year(as.POSIXct(Sys.time(), tz="Asia/Taipei")), index_date, sep = "-")) %>%
  mutate(index_date = ymd_hm(index_date)) %>%
  mutate(index_wday = lubridate::wday(index_date)) %>% 
  mutate(id = row_number()) %>%
  select(index_date, index_board, index_title, index_excerpt, index_author, index_meta, index_link, index_wday, id)

### Take a glimpse of the index dataframe
df_index %>% head(5)

### Preceding operation: use purrr:possibly() to capture side effects
p_read_html <- possibly(read_html, otherwise = NULL)

### Preceding operation: create the dataframe/list
df_article <- tibble(article_link='', article_board='', article_title='', article_author='', article_author_school='', article_text='', article_love='', article_reply='', article_date="") %>% filter(row_number()<1)
df_comment <- list()

### Crawl: use for loop and purrr:map() to parse info
for(i in 1:2) {
  
  # Create counter variable
  j = (i*10) - 9
  k = j + 9
  
  # Use p_read_html() and map() to parse the raw url
  html_raw = df_index[j:k,] %>% pull(index_link) %>% 
    map(function(x){x %>% p_read_html()}) %>% 
    set_names(pull(df_index[j:k,"id"])) %>% compact()
  
  # Use is.na() to avoid error such that the content have been removed
  html_raw_index <- html_raw %>% 
    map(function(x){x %>% html_nodes(".Post_title_2O-1el") %>% html_text()}) %>%
    map_lgl(function(x){!is.na(x)})
   
  html_raw_f <- html_raw[html_raw_index]
  
  # Parse the article part
  article_id <- names(html_raw_f)
  article_title <- html_raw_f %>% 
    map(function(x){x %>% html_nodes(".Post_title_2O-1el") %>% html_text()})
  article_author <- html_raw_f %>% 
    map(function(x){x %>% html_nodes(".PostHeader_uid_3g_pzg") %>% html_text()}) %>%
    map(function(x){if(length(x)==0) x = NA_character_ else(x)})
  article_author_school <- html_raw_f %>% 
    map(function(x){x %>% html_nodes(".PostHeader_author_3AAMDh .PostAuthor_root_3vAJfe") %>% html_text()}) %>%
    map(function(x){if(length(x)==0) x = NA_character_ else(x)})
  article_board <- html_raw_f %>% 
    map(function(x){x %>% html_nodes(".Post_forum_1YYMfp") %>% html_text()})
  article_text <- html_raw_f %>% 
    map(function(x){x %>% html_nodes(".Post_root_23_VRn") %>% html_text()})
  article_love <- html_raw_f %>% 
    map(function(x){x %>% html_nodes(".jTOuHc") %>% html_text()})
  article_reply <- html_raw_f %>% 
    map(function(x){x %>% html_nodes(".DJrdA") %>% html_text()})
  article_date <- html_raw_f %>% 
    map(function(x){x %>% html_nodes(".Post_date_2ipeYS") %>% html_text()})
  
  df_article_tmp <- tibble(article_board=unlist(article_board),
                           article_title=unlist(article_title),
                           article_author=unlist(article_author),
                           article_author_school=unlist(article_author_school),
                           article_text=unlist(article_text),
                           article_love=unlist(article_love),
                           article_reply=unlist(article_reply),
                           article_date=unlist(article_date),
                           article_id = article_id)
  
  # Parse the comment part
  comment_floor <-  html_raw_f %>% 
    map(function(x){x %>% html_nodes(".CommentEntry_floor_VtDUyr") %>% html_text()})
  comment_author_school <-  html_raw_f %>% #.PostAuthorHeader_author_3O30xu span
    map(function(x){x %>% html_nodes(".CommentEntry_author_2BO0i1 .PostAuthorHeader_author_3O30xu span") %>% html_text()})
  comment_love <-  html_raw_f %>% 
    map(function(x){x %>% html_nodes(".jnzHNd") %>% html_text()})
  comment_text <-  html_raw_f %>% 
    map(function(x){x %>% html_nodes(".CommentEntry_content_1ATrw1") %>% html_text()})
  comment_time <-  html_raw_f %>% 
    map(function(x){x %>% html_nodes(".CommentEntry_date_2E4vYF") %>% html_text()})
  
  df_comment_tmp <- pmap(list(comment_floor,comment_author_school,comment_text,comment_time),function(a,b,c,d)
    {bind_cols(comment_floor=unlist(a),comment_author_school=unlist(b),comment_text=unlist(c),comment_time=unlist(d))})
  
  # Bind data from each for loop 
  df_comment <- c(df_comment, df_comment_tmp)
  df_article <- df_article %>% bind_rows(df_article_tmp)
  
  # Update status and let the crawler stop
  print(i)
  tmsleep <- sample(3:5,1)
  Sys.sleep(tmsleep)
}

### Close unnecessary connections
#closeAllConnections()
#gc()

### Take a glimpse of the comments structure
df_comment %>% head(3) %>% str()

### Turn comments list into a dataframe and add a new column "id"
df_comment_bind <- df_comment %>% bind_rows(.id = "id")

### Join the article dataframe and the comments dataframe
df_full <- df_article %>% left_join(df_comment_bind, by = c("article_id"="id"))

### Check the output
df_full %>% tail(5)

### Compare with the index dataframe
df_index %>% tail(5)

library(googledrive)

### Label "new" if the articles were posted later than 22:00 yesterday
df_index_newtag <- df_index %>%
  mutate(new_tag = if_else(index_date >= as.POSIXct(ymd_hms(str_c(ymd(Sys.Date()-1), "10:00:00")), tz="Asia/Taipei"), "new", "old")) %>%
  mutate(hyperlink = str_c('=HYPERLINK("',index_link,'","',index_title,'")')) %>%
  select(-id)

### Export data to the local disk
df_index_newtag %>%
  add_row(index_date = ymd_hms(str_c(ymd(Sys.Date()), "10:00:01")),index_board = "=now()", index_title ='=IMPORTRANGE("https://docs.google.com/spreadsheets/OOO/","dcard_new!A2")') %>%
  arrange(desc(index_date)) %>%
  select(-index_wday) %>%
  write_csv("df_article.csv")

### Pre create a file "cralwer" and a empty gsheet "crawler_dcard"
### Get gsheet from gdrive and update new csv
# gd_crawler_dcard <- drive_get("~/crawler/crawler_dcard")
# gd_crawler_dcard %>% drive_update(media = "df_article.csv")

library(gmailr)
library(glue)
library(tableHTML)

### Below is a demo code since it needs credentials to run

### Read the credentials
use_secret_file("gmailr-zara.json")

### Filter "new" articles
df_index_email <- df_index_newtag %>%
  filter(new_tag == "new") %>%
  select(-c(new_tag, index_wday)) %>%
  mutate(hyperlink = str_sub(index_link, 1, str_locate(index_link, "-")[,1]-1))

### Turn the articles dataframe into a html table
msg = tableHTML(df_index_email)
html_bod <- str_c("<p> Good Morning! There are ",dim(df_index_email)[1], " articles total. </p>", msg)

### Send the email
mime() %>%
  to("MKT_HYHsiang@gmail.com") %>%
  from("BOSS_HYHsiang@gmail.com") %>%
  subject(str_c("ZARA_morning_monitor_", ymd(Sys.Date()))) %>% 
  html_body(html_bod) %>% 
  send_message()