NFL_Web_Scraping_update.Rmd

---
title: "NFL_Web_Scraping"
author: "Daniel Baller"
date: "AUG 28, 2023"
output: pdf_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

## NFL Scores from cbssports.com

```{r}
#Loading Packages
library(tidyverse)
library(stringr)
library(XML)
library(xml2)
library(purrr)
library(rvest)

# Only thing you need to change (update current week)
current_week = 18

################################ Scraping Scores ##############################

week = as.list(seq(1:current_week)) # List of weeks

url = "https://www.cbssports.com/nfl/schedule/2023/regular/"  #base URL

list_of_pages = function(x){
  str_c(url,x)
}

scores = function(week, url){
  
  stats = url%>%
    read_html()%>%
    html_nodes(".CellGame a")%>%
    html_text() %>% 
    matrix(ncol = 1, byrow = TRUE) %>%
    as.data.frame()%>%
    rename(`Score` = V1) %>% 
    separate(Score, sep = " ", into = c("TM1","Score1", "vs", "TM2", "Score2")) %>% 
    mutate(week = week)
}

week%>%
  map(list_of_pages) %>% 
  map2(week,.,scores)%>%
  bind_rows() %>% 
  write_csv(glue::glue("./CSV_Data_Files/NFL_Scores_{length(week)}.csv"))

##################### Scraping CBS Weekly Expert Predictions #########################
url_cbs = "https://www.cbssports.com/nfl/picks/experts/straight-up/"  #base URL

list_of_pages_cbs = function(x){
  str_c(url_cbs,x)
}

cbs_week = function(week, url_cbs){
  
  stats = url_cbs%>%
    read_html()%>%
    html_nodes(".TableExpertPicks-weekRecords")%>%
    html_text() %>% 
    matrix(ncol = 1, byrow = TRUE) %>%
    as.data.frame()%>%
    rename(`Record` = V1) %>% 
    str_extract_all("\\d{1,2}-\\d{1,2}-\\d{1}") %>% 
    unlist() %>% 
    str_split_fixed("-", n=3) %>% 
    as.data.frame() %>% 
    rename(`Correct` = V1, `Wrong` = V2, `Tie` = V3)%>% 
    mutate(Correct = as.numeric(Correct), Wrong = as.numeric(Wrong), Tie = as.numeric(Tie)) %>% 
    mutate(Percent = Correct/(Correct+Wrong+Tie)) %>% 
    mutate(week = week)
}

week%>%
  map(list_of_pages_cbs) %>% 
  map2(week,.,cbs_week)%>%
  bind_rows() %>% 
  write_csv(glue::glue("./CSV_Data_Files/CBS_Experts_{length(week)}.csv"))

###################### Scraping CBS Season Expert Predictions #########################
cbs_season = function(week, url_cbs){
  
  stats = url_cbs%>%
    read_html()%>%
    html_nodes(".TableExpertPicks-overallRecords")%>%
    html_text() %>% 
    matrix(ncol = 1, byrow = TRUE) %>%
    as.data.frame()%>%
    rename(`Record` = V1) %>% 
    str_extract_all("\\d{1,3}-\\d{1,3}-\\d{1}") %>% 
    unlist() %>% 
    str_split_fixed("-", n=3) %>% 
    as.data.frame() %>% 
    rename(`Correct` = V1, `Wrong` = V2, `Tie` = V3)%>% 
    mutate(Correct = as.numeric(Correct), Wrong = as.numeric(Wrong), Tie = as.numeric(Tie)) %>% 
    mutate(Percent = Correct/(Correct+Wrong+Tie)) %>% 
    mutate(week = week)
}

week%>%
  map(list_of_pages_cbs) %>% 
  map2(week,.,cbs_season)%>%
  bind_rows() %>% 
  write_csv(glue::glue("./CSV_Data_Files/CBS_Experts_Season_{length(week)}.csv"))

#################### Scraping ESPN Weekly Expert Predictions #########################
url_espn = "https://www.espn.com/nfl/picks/_/seasontype/2/week/"  #base URL

list_of_pages_espn = function(x){
  str_c(url_espn,x)
}

espn_week = function(week, url_espn){
  
  stats = url_espn%>%
    read_html()%>%
    html_nodes(".wrap-cell span:nth-child(1)")%>%
    html_text() %>% 
    matrix(ncol = 1, byrow = TRUE) %>%
    as.data.frame()%>%
    rename(`Record` = V1) %>% 
    mutate(Correct = str_extract(Record, "(\\d)+(?=-)")) %>% 
    mutate(Wrong = str_extract(Record, "(?<=-)(\\d)+")) %>% 
    mutate(Correct = as.numeric(Correct), Wrong = as.numeric(Wrong)) %>% 
    mutate(Percent = Correct/(Correct+Wrong)) %>% 
    filter(Percent!="NaN") %>% 
    mutate(week = week)
}

week%>%
  map(list_of_pages_espn) %>% 
  map2(week,.,espn_week)%>%
  bind_rows() %>% 
  write_csv(glue::glue("./CSV_Data_Files/ESPN_Experts_{length(week)}.csv"))

################## Scraping ESPN Season Expert Predictions #########################
espn_season = function(week, url_espn){
  
  stats = url_espn%>%
    read_html()%>%
    html_nodes(".wrap-cell span+ span")%>%
    html_text() %>% 
    matrix(ncol = 1, byrow = TRUE) %>%
    as.data.frame()%>%
    rename(`Record` = V1) %>% 
    mutate(Correct = str_extract(Record, "(\\d)+(?=-)")) %>% 
    mutate(Wrong = str_extract(Record, "(?<=-)(\\d)+")) %>% 
    mutate(Correct = as.numeric(Correct), Wrong = as.numeric(Wrong)) %>% 
    mutate(Percent = Correct/(Correct+Wrong)) %>% 
    filter(Percent!="NaN") %>% 
    mutate(week = week)
}

week%>%
  map(list_of_pages_espn) %>% 
  map2(week,.,espn_season)%>%
  bind_rows() %>% 
  write_csv(glue::glue("./CSV_Data_Files/ESPN_Experts_Season_{length(week)}.csv"))
```

################## STOP UPDATE CURRENT WEEK TO NEXT WEEK FIRST ########################

#Scraping Moneyline odds Ceasars Sports Book
```{r}
week_odds = current_week+1

read_html("https://www.sportsline.com/nfl/odds/money-line/")%>%
    html_nodes(".EthnK .cQwbsx , .projected-score+ td .primary")%>%
    html_text() %>% 
    matrix(ncol = 2, byrow = TRUE) %>%
    as.data.frame() %>% 
    mutate(V2 = as.numeric(V2)) %>% 
    rename(`team` = V1, `odds` = V2) %>% 
    add_column(week = week_odds) %>% 
    write_csv(glue::glue("./CSV_Data_Files/Moneyline_Odds_{week_odds}.csv"))
```



########################## Sandbox
```{r}
read_html("https://www.sportsline.com/nfl/odds/money-line/")%>%
    html_nodes("td:nth-child(4) .primary , h4")%>%
    html_text() %>% 
    matrix(ncol = 4, byrow = TRUE) %>%
    as.data.frame() %>% 
    mutate(V2 = as.numeric(V2), V4 = as.numeric(V4)) %>% 
    rename(`away_team` = V1, `away_odds` = V2, `home_team` = V3, `home_odds` = V4) %>% 
    add_column(week = week[[length(week)]])
 

read_html("https://www.cbssports.com/nfl/picks/experts/straight-up/17/")%>%
    html_nodes(".TableExpertPicks-overallRecords")%>%
    html_text() %>% 
    matrix(ncol = 1, byrow = TRUE) %>%
    as.data.frame()%>%
    rename(`Record` = V1) %>% 
    str_extract_all("\\d{1,3}-\\d{1,3}-\\d{1}") %>% 
    unlist() %>% 
    str_split_fixed("-", n=3) %>% 
    as.data.frame() %>% 
    rename(`Correct` = V1, `Wrong` = V2, `Tie` = V3)%>% 
    mutate(Correct = as.numeric(Correct), Wrong = as.numeric(Wrong), Tie = as.numeric(Tie)) %>% 
    mutate(Percent = Correct/(Correct+Wrong+Tie))
```


```{r}
read_html("https://www.espn.com/nfl/picks/_/seasontype/2/week/3")%>%
    html_nodes(".wrap-cell span:nth-child(1)")%>%
    html_text() %>% 
    matrix(ncol = 1, byrow = TRUE) %>%
    as.data.frame()%>%
    rename(`Record` = V1) %>% 
    mutate(Correct = str_extract(Record, "(\\d)+(?=-)")) %>% 
    mutate(Wrong = str_extract(Record, "(?<=-)(\\d)+")) %>% 
    mutate(Correct = as.numeric(Correct), Wrong = as.numeric(Wrong)) %>% 
    mutate(Percent = Correct/(Correct+Wrong)) %>% 
    filter(Percent!="NaN")
```

```{r}
read_html("https://www.cbssports.com/nfl/schedule/2020/regular/1/")%>%
    html_nodes(".CellGame a")%>%
    html_text() %>% 
    matrix(ncol = 1, byrow = TRUE) %>%
    as.data.frame()%>%
    rename(`Score` = V1) %>% 
    separate(Score, sep = " ", into = c("TM1","Score1", "vs", "TM2", "Score2"))
```