Harvesting Real Estate

The Variables

We’ll be using a particular set of variables which I won’t be displaying here, sorry if that makes the code more difficult to understand.

# Please respect the privacy of the website
# 
# This function calls identifiable variable of the website

if(!exists("hid_var", mode = "function")) {
  source("hid_var.R") 
}
hid_var()

Web scraping with rvest involves:

1. reading the entire page as html with read_html

• If you view the page’s source, you will see the html code which this function will be reading. For Chrome & Edge you can currently do this by right clicking and selecting “view page source”.

2. choosing the html tags & elements of interest with html_elements

• Within the html code are elements, usually denoted with some start and end tags <...> … </...>. html_elements allows us to select certain tags to collect their contents (the element) using CSS selectors, or XPath expressions.

  For my purpose, I used selector gadget to find the CSS selector I was interested in.

3. choosing additional information of the html tags / elements in the form of attributes with html_attr

• The attribute is found in the start tag and can be used to isolate additional details. In the example, <a href="https://example.com">An Example Site</a>. The tag is <a> which has the attribute href.

They are other functions, but we’ll only be using these ones.

We will write our code to read multiple pages which will be implemented using a function to read each page.

We’ll open a connection to each page, read each page, and then close our connection to it on each iteration to ensure this works without hiccups.

We then combine the results from each page using map_dfr which will create one combined dataframe by appending each page’s results by row.

# Get the website Data


# Download property to find pages to scrape
temp_property_id <- map_dfr(1:row_limit, function(i) {
  
  # simple and effective progress indicator
  # cat(".")
  
  # download the page of interest
  pg <- paste0(page_begin,
               i,
               page_end) %>%
    url() %>%
    read_html()
  
  # close connection to the page of interest
  paste0(page_begin,
         i,
         page_end) %>%
    url() %>%
    close()
  
  # Check lengths of the three points of interest
  length_1 <- length(i)
  
  length_2 <- pg %>%
    html_elements(temp_1) %>%
    html_text() %>%
    length()
  
  length_3 <- pg %>%
    html_elements("a") %>%
    html_attr("onclick") %>%
    as_tibble() %>%
    filter(str_detect(string = value,
                      pattern = "LoadMap")) %>%
    as_vector() %>%
    length()
  
  # Select the max length between the three points of interest
  max_length <- max(c(length_1, length_2, length_3))
  
  # Place points of interest in a dataframe, and repeat a dummy value until they are the same length as the max length to avoid mismatched columns
  data.frame(
    web_page = c(i,
                 rep(i, max_length - length_1)),
    property_id = c(
      pg %>%
        html_elements(temp_1) %>%
        html_attr("onclick") %>%
        parse_number(),
      rep(i, max_length - length_2)),
    location = c(
      pg %>%
        html_elements("a") %>%
        html_attr("onclick") %>%
        as_tibble() %>%
        filter(str_detect(string = value,
                          pattern = "LoadMap")) %>%
        as_vector(),
      rep(i, max_length - length_3)),
    stringsAsFactors=FALSE)
})

# Display five random rows in generated dataframe
sample(x = 1:nrow(temp_property_id),
       size = 5,
       replace = FALSE) %>%
  temp_property_id[.,] %>%
  #obscure variable to use for identification
  mutate(property_id = "00000",
         location = paste( head(strsplit(location, '')[[1]], 14), collapse = "")
         ) %>%
  as_tibble() %>%
  gt::gt() %>% 
  gt::tab_header(
    title = "Sample Values",
    subtitle = "Sample taken from initial scraped dataset"
  )

We’ll clean this dataframe by isolating the latitude and longitude where possible, and doing a quick pass to remove coordinates which don’t fall in a certain distance around Barbados.

temp_property_id <- temp_property_id %>%
  separate(col = location,
           into = c("latitude", "longitude", NA),
           sep = ",") %>%
  # Simple sanity check for the values for latitude & longitude
  mutate(latitude = parse_number(latitude),
         latitude = if_else(latitude > 13.044347 & latitude < 13.335841,
                            latitude,
                            NA_real_),
         longitude = parse_number(longitude),
         longitude = if_else(longitude > -59.647765 & longitude < -59.410176,
                             longitude,
                             NA_real_))

## Warning: Expected 3 pieces. Additional pieces discarded in 5940 rows [1, 2, 3,
## 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].

## Warning: 1348 parsing failures.
##  row col expected     actual
## 3716  -- a number LoadMap(''
## 3717  -- a number LoadMap(''
## 3732  -- a number LoadMap(''
## 3733  -- a number LoadMap(''
## 3735  -- a number LoadMap(''
## .... ... ........ ..........
## See problems(...) for more details.

## Warning: 1348 parsing failures.
##  row col expected actual
## 3716  -- a number     ''
## 3717  -- a number     ''
## 3732  -- a number     ''
## 3733  -- a number     ''
## 3735  -- a number     ''
## .... ... ........ ......
## See problems(...) for more details.

# Display five random rows in generated dataframe
sample(x = 1:nrow(temp_property_id),
       size = 5,
       replace = FALSE) %>%
  temp_property_id[.,] %>%
  #obscure variable to use for identification
  mutate(property_id = "00000",
         latitude = round(latitude, 2),
         longitude = round(longitude, 2)
         ) %>%
  as_tibble() %>%
  gt::gt() %>% 
  gt::tab_header(
    title = "Sample Values",
    subtitle = "Sample taken from initial scraped dataset"
  )

We will use the property id to read each of the pages we’re interested in. On each page, our points of interest do not always appear, and when they do appear, they don’t always appear in the same order.

We will accommodate for this by ensuring our code will insert a ‘NA’ when the point of interest isn’t found and reorganising the collected data to properly group the information.

The only new function added is the use of html_text which allows us to take the text from an element.

if (!any( str_detect(list.files(full.names = TRUE,
                                recursive = TRUE,
                                include.dirs = TRUE),
                     "realestate.RData")
)
){
  
  raw_length <- nrow(temp_property_id)
  
  # Scrape the relevant details
  raw_data <- map_dfr(1:raw_length, function(i) {
    # simple but effective progress indicator
    cat(".")
    
    pg <- read_html(url(paste0(raw_page_begin,
                               temp_property_id$property_id[i],
                               raw_page_end)))
    
    close(url(paste0(raw_page_begin,
                     temp_property_id$property_id[i],
                     raw_page_end)))
    
    # the ifelse checks if the desired value exists, if yes, take the value, if no insert NA
    data.frame(
      row_1 = ifelse(
        length(html_text(html_elements(pg, type_1))) == 0,
        NA_character_,
        html_text(html_elements(pg, type_1))
      ),
      row_2 = ifelse(
        length(html_text(html_elements(pg, type_2))) == 0,
        NA_character_,
        html_text(html_elements(pg, type_2))
      ),
      row_3 = ifelse(
        length(html_text(html_elements(pg, type_3))) == 0,
        NA_character_,
        html_text(html_elements(pg, type_3))
      ),
      row_4 = ifelse(
        length(html_text(html_elements(pg, type_4))) == 0,
        NA_character_,
        html_text(html_elements(pg, type_4))
      ),
      row_5 = ifelse(
        length(html_text(html_elements(pg, type_5))) == 0,
        NA_character_,
        html_text(html_elements(pg, type_5))
      ),
      row_6 = ifelse(
        length(html_text(html_elements(pg, type_6))) == 0,
        NA_character_,
        html_text(html_elements(pg, type_6))
      ),
      row_7 = ifelse(
        length(html_text(html_elements(pg, type_7))) == 0,
        NA_character_,
        html_text(html_elements(pg, type_7))
      ),
      row_0 = ifelse(
        length(html_text(html_elements(pg,type_8))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_8))
      ),
      title_1 = ifelse(
        length(html_text(html_elements(pg,type_9))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_9))
      ),
      title_2 = ifelse(
        length(html_text(html_elements(pg,type_10))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_10))
      ),
      title_3 = ifelse(
        length(html_text(html_elements(pg,type_11))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_11))
      ),
      title_4 = ifelse(
        length(html_text(html_elements(pg,type_12))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_12))
      ),
      title_5 = ifelse(
        length(html_text(html_elements(pg,type_13))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_13))
      ),
      title_6 = ifelse(
        length(html_text(html_elements(pg,type_14))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_14))
      ),
      title_7 = ifelse(
        length(html_text(html_elements(pg,type_15))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_15))
      ),
      title_0 = ifelse(
        length(html_text(html_elements(pg,type_16))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_16))
      ),
      description = ifelse(
        length(html_text(html_elements(pg,type_a))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_a))
      ),
      realtor = ifelse(
        length(html_text(html_elements(pg,type_b))) == 0,
        NA_character_,
        html_text(html_elements(pg,type_b))
      ),
      blurb = ifelse(
        length(html_text(html_elements(pg,type_17))) == 0,
        NA_character_,
        # This uses `paste`, allowing us to combine multiple rows into one string
        paste(html_text(html_elements(pg,type_17)), collapse = " ")
      ),
      # We collected these values earlier, so we won't need to insert NA here
      latitude = temp_property_id$latitude[i],
      longitude = temp_property_id$longitude[i],
      link = paste0(raw_page_begin,
                    temp_property_id$property_id[i],
                    raw_page_end),
      stringsAsFactors=FALSE)
    
  })
  
  
  # Display five random rows in generated dataframe
  sample(x = 1:nrow(raw_data),
         size = 5,
         replace = FALSE) %>%
    raw_data[.,] %>%
  # remove variables which may identify locations
  select(!(description:link)) %>%
    as_tibble() %>%
    gt::gt() %>% 
  gt::tab_header(
    title = "Sample Values",
    subtitle = "Sample taken from scraped dataset"
  )
  
}

We will clean this dataset by sorting the data into appropriate columns and pull some useful features from the collected unstructured text which we labelled blurb.

# Clean the Raw Data
# I currently like using case_when here because it's easy for me to read and I can easily edit as desired.
# If you use a more efficient method, let me know.

if (!any( str_detect(list.files(full.names = TRUE,
                                recursive = TRUE,
                                include.dirs = TRUE),
                     "realestate.RData")
)
){
  
  data <- raw_data %>%
    mutate(
      # Collect the price into one column
      price = case_when(
        title_0 == "Sale Price" ~ row_0,
        title_1 == "Sale Price" ~ row_1,
        title_2 == "Sale Price" ~ row_2,
        title_3 == "Sale Price" ~ row_3,
        title_4 == "Sale Price" ~ row_4,
        title_5 == "Sale Price" ~ row_5,
        title_6 == "Sale Price" ~ row_6,
        title_7 == "Sale Price" ~ row_7,
        TRUE                    ~ NA_character_
      ),
      # Collect the property status into one column
      property_status = case_when(
        title_0 == "Property Status" ~ row_0,
        title_1 == "Property Status" ~ row_1,
        title_2 == "Property Status" ~ row_2,
        title_3 == "Property Status" ~ row_3,
        title_4 == "Property Status" ~ row_4,
        title_5 == "Property Status" ~ row_5,
        title_6 == "Property Status" ~ row_6,
        title_7 == "Property Status" ~ row_7,
        TRUE                         ~ NA_character_
      ),
      # Collect the property type into one column
      property_type = case_when(
        title_0 == "Property Type" ~ row_0,
        title_1 == "Property Type" ~ row_1,
        title_2 == "Property Type" ~ row_2,
        title_3 == "Property Type" ~ row_3,
        title_4 == "Property Type" ~ row_4,
        title_5 == "Property Type" ~ row_5,
        title_6 == "Property Type" ~ row_6,
        title_7 == "Property Type" ~ row_7,
        TRUE                       ~ NA_character_
      ),
      # Collect the number of bedrooms into one column
      bedroom_no = case_when(
        title_0 == "Bedrooms" ~ row_0,
        title_1 == "Bedrooms" ~ row_1,
        title_2 == "Bedrooms" ~ row_2,
        title_3 == "Bedrooms" ~ row_3,
        title_4 == "Bedrooms" ~ row_4,
        title_5 == "Bedrooms" ~ row_5,
        title_6 == "Bedrooms" ~ row_6,
        title_7 == "Bedrooms" ~ row_7,
        TRUE                  ~ NA_character_
      ),
      # Collect the number of bathrooms into one column
      bathroom_no = case_when(
        title_0 == "Bathrooms" ~ row_0,
        title_1 == "Bathrooms" ~ row_1,
        title_2 == "Bathrooms" ~ row_2,
        title_3 == "Bathrooms" ~ row_3,
        title_4 == "Bathrooms" ~ row_4,
        title_5 == "Bathrooms" ~ row_5,
        title_6 == "Bathrooms" ~ row_6,
        title_7 == "Bathrooms" ~ row_7,
        TRUE                   ~ NA_character_
      ),
      # Collect the year built into one column
      year_built = case_when(
        title_0 == "Year Built" ~ row_0,
        title_1 == "Year Built" ~ row_1,
        title_2 == "Year Built" ~ row_2,
        title_3 == "Year Built" ~ row_3,
        title_4 == "Year Built" ~ row_4,
        title_5 == "Year Built" ~ row_5,
        title_6 == "Year Built" ~ row_6,
        title_7 == "Year Built" ~ row_7,
        TRUE                    ~ NA_character_
      ),
      # Collect the building area into one column
      bld_area = case_when(
        title_0 == "Building Area" ~ row_0,
        title_1 == "Building Area" ~ row_1,
        title_2 == "Building Area" ~ row_2,
        title_3 == "Building Area" ~ row_3,
        title_4 == "Building Area" ~ row_4,
        title_5 == "Building Area" ~ row_5,
        title_6 == "Building Area" ~ row_6,
        title_7 == "Building Area" ~ row_7,
        TRUE                       ~ NA_character_
      ),
      # Collect the property area into one column
      pty_area = case_when(
        title_0 == "Property Area" ~ row_0,
        title_1 == "Property Area" ~ row_1,
        title_2 == "Property Area" ~ row_2,
        title_3 == "Property Area" ~ row_3,
        title_4 == "Property Area" ~ row_4,
        title_5 == "Property Area" ~ row_5,
        title_6 == "Property Area" ~ row_6,
        title_7 == "Property Area" ~ row_7,
        TRUE                       ~ NA_character_
      ),
      # Create feature for beach access
      beach_access = case_when(
        str_detect(blurb, regex("Beach", ignore_case = T)) ~ TRUE,
        TRUE                                               ~ FALSE
      ),
      # Create feature for pool access
      pool_access = case_when(
        str_detect(blurb, regex("Pool", ignore_case = T)) ~ TRUE,
        TRUE                                              ~ FALSE
      ),
      # Create feature for golf access
      golf_access = case_when(
        str_detect(blurb, regex("Golf", ignore_case = T)) ~ TRUE,
        TRUE                                              ~ FALSE
      ),
      # Create feature for furnished building
      furnished = case_when(
        str_detect(blurb, regex("Furnish", ignore_case = T)) ~ TRUE,
        TRUE                                                 ~ FALSE
      ),
      # Create feature for currency used
      currency = case_when(
        str_detect(price,"(BB)") ~ "BBD",
        str_detect(price,"(US)") ~ "USD",
        TRUE ~ NA_character_),
      # Create feature for rent availability
      rent_available = case_when(
        str_detect(price,regex("\\s(rent)|^(rent)|\\s(lease)|^(lease)", ignore_case = T)) ~ TRUE,
        TRUE ~ FALSE),
      # Create feature for location by Parish
      parish = case_when(
        str_detect(description, "Christ Church") ~ "Christ Church",
        str_detect(description, "St. Andrew")    ~ "St. Andrew",
        str_detect(description, "St. George")    ~ "St. George",
        str_detect(description, "St. James")     ~ "St. James",
        str_detect(description, "St. John")      ~ "St. John",
        str_detect(description, "St. Joseph")    ~ "St. Joseph",
        str_detect(description, "St. Lucy")      ~ "St. Lucy",
        str_detect(description, "St. Michael")   ~ "St. Michael",
        str_detect(description, "St. Peter")     ~ "St. Peter",
        str_detect(description, "St. Philip")    ~ "St. Philip",
        str_detect(description, "St. Thomas")    ~ "St. Thomas",
        TRUE                                     ~ NA_character_
      ),
      # Parse the values which should be numbers
      price = parse_number(price),
      # Convert any number which was found as acres to sq. ft.
      pty_area = case_when(
        str_detect(pty_area,
                   regex("acre",
                         ignore_case = T)) ~ parse_number(pty_area) * 43560.04,
        str_detect(blurb,
                   regex("[:digit:](|\\.[:digit:])\\s+(acre)",
                         ignore_case = T)) ~  parse_number(
                           str_extract(blurb,
                                       regex("[:digit:](|\\.[:digit:])\\s+(acre)",
                                             ignore_case = T))
                         ) * 43560.04,
        TRUE ~ parse_number(pty_area)
      ),
      bld_area = parse_number(bld_area),
      bedroom_no = parse_number(bedroom_no),
      bathroom_no = parse_number(bathroom_no),
      year_built = parse_number(year_built),
      # Create the feature for price per square foot
      # We make some basic assumptions about what will likely be a price per square foot
      price_per_sqft = case_when(
        price < 100 ~ price,
        price > 100 & price < 5000 ~ NA_real_,
        price > 5000 & property_type == "Land" & !is.na(pty_area) ~ price / pty_area,
        price > 5000 & property_type == "Land" & !is.na(bld_area) ~ price / bld_area,
        price > 5000 & property_type != "Land" & !is.na(bld_area) ~ price / bld_area,
        price > 5000 & property_type != "Land" & !is.na(pty_area) ~ price / pty_area,
        TRUE ~ NA_real_
      ),
      # Remove any infinity values
      price_per_sqft = ifelse(is.infinite(price_per_sqft), NA_real_, price_per_sqft)
    ) %>%
    select(
      price,
      price_per_sqft,
      currency,
      property_status,
      property_type,
      rent_available,
      bedroom_no,
      bathroom_no,
      beach_access,
      pool_access,
      golf_access,
      furnished,
      bld_area,
      pty_area,
      year_built,
      realtor,
      latitude,
      longitude,
      parish,
      description,
      link
    )
  data <- na_if(data, "-")
  
} else{
  files <- list.files(full.names = TRUE,
                      recursive = TRUE,
                      include.dirs = TRUE)
  
  # Select Desired folder
  used_file <- files[str_detect(files, "realestate.RData")]
  
  load(used_file)
  
  rm(files,
     used_file)
}

# Display five random rows in generated dataframe
sample(x = 1:nrow(data),
       size = 5,
       replace = FALSE) %>%
  data[.,] %>%
  # remove variables which may identify locations
  select(!(realtor:link)) %>%
  as_tibble() %>%
  gt::gt() %>% 
  gt::tab_header(
    title = "Sample Values",
    subtitle = "Sample taken from cleaned dataset"
  ) %>%
  gt::fmt_number(
    columns = c(price_per_sqft,
                bedroom_no,
                bathroom_no,
                bld_area,
                pty_area),
    decimals = 0,
    suffixing = TRUE
  ) %>%
  gt::fmt_currency(
    columns = price,
    suffixing = TRUE,
    currency = "BBD"
  )

We’ll save our dataframe for use at a later date.

# Save file
if (!any( str_detect(list.files(full.names = TRUE,
                                recursive = TRUE,
                                include.dirs = TRUE),
                     "realestate.RData")
)
) { save(data, file = "realestate.RData")
}