Press the Show button below to reveal the code.
# Set global options for all code chunks
knitr::opts_chunk$set(
# Disable messages printed by R code chunks
message = FALSE,
# Disable warnings printed by R code chunks
warning = FALSE,
# Show R code within code chunks in output
echo = TRUE,
# Include both R code and its results in output
include = TRUE,
# Evaluate R code chunks
eval = TRUE,
# Enable caching of R code chunks for faster rendering
cache = FALSE,
# Align figures in the center of the output
fig.align = "center",
# Enable retina display for high-resolution figures
retina = 2,
# Show errors in the output instead of stopping rendering
error = TRUE,
# Do not collapse code and output into a single block
collapse = FALSE
)
# Start the figure counter
fig_count <- 0
# Define the captioner function
captioner <- function(caption) {
fig_count <<- fig_count + 1
paste0("Figure ", fig_count, ": ", caption)
}
# Define the function to truncate a number to two decimal places
truncate_to_two <- function(x) {
floor(x * 100) / 100
}
# Load required libraries
library(httr)
library(jsonlite)
library(readr)
library(dplyr)
library(zoo) # to use na.locf
library(tidyr) # to use pivot_wider
library(slackr)
library(grateful)
library(here)
library(DT)
initial_date <- as.Date("2025-04-23")
source(here("keys.R"))
slackr_setup(token = token)
## [1] "Successfully connected to Slack"
format_market_cap <- function(x) {
sapply(x, function(val) {
if (is.na(val)) {
return("NA")
}
val <- as.numeric(val)
if (val >= 1e12) {
return(sprintf("%.1fT", val / 1e12))
} else if (val >= 1e9) {
return(sprintf("%.1fB", val / 1e9))
} else if (val >= 1e6) {
return(sprintf("%.1fM", val / 1e6))
} else if (val >= 1e3) {
return(sprintf("%.1fk", val / 1e3))
} else {
return(as.character(val))
}
})
}
Data is downloaded daily from here.
The lastsale
value reported in the previous link is the
open price when the market closes, according to this website.
Data collection started on April 23, 2025.
aux_day <- Sys.Date()
day_of_week <- weekdays(aux_day)
if (day_of_week == "Monday") {
today <- aux_day - 3
} else if (day_of_week %in% c("Tuesday", "Wednesday", "Thursday", "Friday")) {
today <- aux_day - 1
} else {
today <- aux_day # or handle Saturday/Sunday however you want
}
# Define the API URL
url <- "https://api.nasdaq.com/api/screener/stocks?tableonly=true&limit=25&offset=0&download=true"
# Set headers to mimic a browser
headers <- add_headers(`User-Agent` = "Mozilla/5.0")
# Send GET request
response <- GET(url, headers)
# Check status
if (status_code(response) == 200) {
# Parse JSON content
content_json <- content(response, "text", encoding = "UTF-8")
data_list <- fromJSON(content_json, flatten = TRUE)
# Extract rows from JSON
stock_data <- data_list$data$rows
# Add current date column
stock_data$date <- today
# Create filename with today's date
file_name <- paste0("nasdaq_screener_", today, ".csv")
# Save to CSV
write_csv(stock_data, here("data", file_name))
cat("✅ Data saved to", file_name, "\n")
} else {
cat("❌ Failed to fetch data. Status code:", status_code(response), "\n")
}
## ✅ Data saved to nasdaq_screener_2025-06-30.csv
# The following line should replace the next following line
today_stock_data <- read_csv(here("data", file_name)) |>
filter(nchar(symbol)<5) |> # filter out stocks with more than 4 characters (not tradeable)
filter(!grepl("\\^", symbol)) |> # remove symbols containing ^
select(-url, -netchange, -pctchange) |> # remove unnecessary columns
mutate(lastsale = as.numeric(gsub("[$,]", "", lastsale))) |># remove $ and , from lastsale and make it numeric
select(symbol, date, lastsale, everything()) # reorder columns
# Remove symbols that are not longer being traded
cummulative_stock_data <- readRDS(here("data/cummulative_stock_data.RDS")) |>
filter(symbol %in% today_stock_data$symbol)
# The following two instructions deals with new_data being added when it is already there.
# This might happen when the stock market is closed during weekdays
# Keep only rows from today_stock_data that are NOT already in cummulative_stock_data
new_today_stock_data <- anti_join(today_stock_data,
cummulative_stock_data,
by = setdiff(names(today_stock_data), "date")) # Identify columns to match (everything except 'date')
clean_cummulative_stock_data <- bind_rows(cummulative_stock_data, new_today_stock_data) |>
arrange(symbol, date) |>
select(symbol, date, lastsale, marketCap, country, industry, sector, ipoyear, volume, name)
# Get unique dates from the dataset
unique_dates <- sort(unique(clean_cummulative_stock_data$date))
# Step 1: Expand each symbol to all existing dates
stock_data_expanded <- clean_cummulative_stock_data |>
group_by(symbol) |>
complete(date = unique_dates) |>
ungroup()
# Step 2: Fill backward for multiple columns
columns_to_fill <- c(
"lastsale", "marketCap", "country", "industry",
"sector", "ipoyear", "volume", "name"
)
stock_data_filled <- stock_data_expanded |>
group_by(symbol) |>
arrange(date, .by_group = TRUE) |>
mutate(across(all_of(columns_to_fill),
~ zoo::na.locf(.x, fromLast = TRUE, na.rm = FALSE))) |> # filling from the last available value backward
mutate(across(all_of(columns_to_fill),
~ zoo::na.locf(.x, fromLast = FALSE, na.rm = FALSE))) |> # filling from the first available value forward
ungroup() |>
arrange(symbol, date)
stock_data_to_save <- stock_data_filled |>
group_by(symbol) |>
arrange(date, .by_group = TRUE) |>
filter(!any(is.na(lastsale))) |>
ungroup() |>
arrange(symbol, date)
saveRDS(stock_data_to_save, here("data/cummulative_stock_data.RDS"))
Let X_k(t_i)
denote the value of a stock
X_k
at day t_i
, where i= 0,...,N
.
Let pct X_k(t_i)
denote the percentage change of
X_k(t_i)
with respect to X_k(t_0)
, i.e.,
pct X_k(t_i) = (X_k(t_i) - X_k(t_0))/X_k(t_0) * 100
. For
each stock X_k
, the filters are as follows:
The first filter is that X_k(t_0) <= X_k(t_N)
,
i.e., the stock must have a positive growth trend within the period of
analysis.
The second filter is that
0.8*X_k(t_{i-1}) <= X_k(t_i) <= 2*X_k(t_{i-1})
, i.e.,
the next day price is at most twice and at least 80% of the last price
(from day to day it can only fall 20%).
The third filter is that
max{ pct X_k(t_0), ..., pct X_k(t_{N-1})} / 2 <= pct X_k(t_N)
,
i.e., the final price growth is at least half of the highest
growth.
The fourth filter is that
min{|pct X_k(t_0)|, ..., |pct X_k(t_{N-1})|} <= pct X_k(t_N)
,
i.e., the stock must not have fallen more than what it has
grown.
# stock_data_to_save <- readRDS(here("data/cummulative_stock_data.RDS"))
stocks_data_with_comment_column <- stock_data_to_save |>
mutate(comment = NA_character_)
# The following ensures that we only keep stocks that have a positive growth trend
stocks_data_grow <- stocks_data_with_comment_column |>
group_by(symbol) |>
arrange(date, .by_group = TRUE) |>
mutate(comment = if_else(
is.na(comment) & !(first(lastsale) <= last(lastsale)),
"did not pass X(t_0) <= X(t_N)",
comment
)) |>
ungroup()
# The following ensures that the next day price is at most twice and at least 80% of the last price (from day to day it can only fall 20%)
stocks_data_window <- stocks_data_grow |>
group_by(symbol) |>
arrange(date, .by_group = TRUE) |>
mutate(
comment = if_else(
is.na(comment) & !all((lead(lastsale) > 0.8 * lastsale) & (lead(lastsale) < 2 * lastsale), na.rm = TRUE),
"did not pass 0.8*X(t_{i-1}) <= X(t_i) <= 2*X(t_{i-1})",
comment
)) |>
ungroup()
# Compute the percentage change from the start date (daily and accumulated)
# Keep only those stocks where the final price growth is at least half of the highest growth (before the end).
stock_data_pct_change <- stocks_data_window |>
group_by(symbol) |>
arrange(date, .by_group = TRUE) |>
mutate(daily_pct_change = (lastsale - lag(lastsale)) / lag(lastsale) * 100,
accum_pct_change = (lastsale - first(lastsale)) / first(lastsale) * 100) |>
mutate(comment = if_else(
is.na(comment) & !((max(head(accum_pct_change, -1)) / 2) <= last(accum_pct_change)),
"did not pass max{pct X(t_0), ..., pct X(t_{N-1})} / 2 <= pct X(t_N)",
comment
)) |>
ungroup()
# Filter symbols that have fallen (anytime) more that what they have grown (at the end)
stock_data_pct_change_that_grow <- stock_data_pct_change |>
group_by(symbol) |>
arrange(date, .by_group = TRUE) |>
mutate(
comment = if_else(
is.na(comment) &
any(accum_pct_change < 0) & !(abs(min(head(accum_pct_change, -1))) <= last(accum_pct_change)),
"did not pass min{|pct X(t_0)|, ..., |pct X(t_{N-1})|} <= pct X(t_N)",
comment
)
) |>
ungroup()
# Removes the first date (the one that was used to compute the daily percentage change)
aux <- stock_data_pct_change_that_grow |>
filter(date != min(date))
AUX_to_rank <- aux |>
filter(is.na(comment))
# From here on there are no more filters
Let R_j
be the rank of the stocks according to their
percentage change pct X_k(t_i)
at day t_i
. The
ranks is computed as follows:
Accumulated rank
. More complicated than the others.
The idea is to compute the rank of each stock at each date, and then
accumulate the ranks according to the following logic:
R_j(t_i) >= R_j(t_{i-1})
, then
R_j(t_i) = R_j(t_{i-1}) + R_j(t_i)
.R_j(t_i) < R_j(t_{i-1})
, then
R_j(t_i) = R_j(t_{i-1}) - R_j(t_i)
.This means that if a stock has a better rank than the previous day, its accumulated rank increases by the value of its current rank. If it has a worse rank, its accumulated rank decreases by the value of its current rank.
Sum of ranks
. This is simply
R_1 + R_2 + ... + R_N
.
Last day rank
. This is simply
R_N
.
# The following contains the information of the surviving symbols
info_of_surviving_symbols <- aux |>
filter(date == max(date)) |> # so we get the last market cap and volume, for example
select(-date, -daily_pct_change) |>
mutate(marketCap_ = format_market_cap(marketCap),
accum_pct_change = round(accum_pct_change, 2))
# The following is the list of symbols that passed all filters
info_of_those_that_passed_all_filters <- info_of_surviving_symbols |>
filter(is.na(comment)) |>
select(-comment)
# The following is the list of symbols that did not pass all filters
reason_for_those_that_did_not_pass_all_filters <- info_of_surviving_symbols |>
filter(!is.na(comment)) |>
select(symbol, comment)
# Compute the rank for each date
aux_ranked <- AUX_to_rank |>
group_by(date) |>
arrange(accum_pct_change) |>
mutate(rank = row_number()) |>
ungroup() |>
arrange(symbol,date)
# Compute last day rank
last_day_rank <- aux_ranked |>
group_by(symbol) |>
filter(date == max(date)) |>
summarise(last_rank = rank) |>
ungroup() |>
arrange(desc(last_rank))
# Compute the sum of ranks
sum_rank <- aux_ranked |>
group_by(symbol) |>
summarise(sum_rank = sum(rank)) |>
ungroup() |>
arrange(desc(sum_rank))
# Create the table of ranks
table_of_ranks <- aux_ranked |>
select(symbol, date, rank) |>
pivot_wider(
names_from = date,
values_from = rank
)
# assume first column is 'symbol'
ranking_accumulated <- table_of_ranks
# Get only the numeric part (the ranks)
rank_matrix <- as.matrix(table_of_ranks[,-1])
# Apply the custom accumulation logic row-wise
accumulated_matrix <- t(apply(rank_matrix, 1, function(row) {
acc <- numeric(length(row))
acc[1] <- row[1]
for (i in 2:length(row)) {
if (row[i] >= row[i - 1]) {
acc[i] <- acc[i - 1] + row[i]
} else {
acc[i] <- acc[i - 1] - row[i]
}
}
return(acc)
}))
# Combine with symbols again
ranking_accumulated[,-1] <- accumulated_matrix
acc_rank <- ranking_accumulated[, c(1,ncol(ranking_accumulated))] |>
rename(acc_rank = colnames(ranking_accumulated)[ncol(ranking_accumulated)]) |>
arrange(desc(acc_rank))
# Augment the ranks with info
aug_acc_rank <- acc_rank |>
inner_join(info_of_those_that_passed_all_filters, by = "symbol") |>
arrange(desc(acc_rank)) |>
select(symbol, acc_rank, marketCap, marketCap_, accum_pct_change, country, industry, sector, ipoyear, volume, lastsale, name) |>
mutate(rank = row_number()) |>
mutate(symbol = paste0("<a href='https://finance.yahoo.com/quote/", symbol, "' target='_blank'>", symbol, "</a>")) |>
select(symbol, acc_rank, marketCap, marketCap_, accum_pct_change, country, industry, sector, ipoyear, volume, rank, lastsale, name)
aug_sum_rank <- sum_rank |>
inner_join(info_of_those_that_passed_all_filters, by = "symbol") |>
arrange(desc(sum_rank)) |>
select(symbol, sum_rank, marketCap, marketCap_, accum_pct_change, country, industry, sector, ipoyear, volume, lastsale, name) |>
mutate(rank = row_number()) |>
mutate(symbol = paste0("<a href='https://finance.yahoo.com/quote/", symbol, "' target='_blank'>", symbol, "</a>")) |>
select(symbol, sum_rank, marketCap, marketCap_, accum_pct_change, country, industry, sector, ipoyear, volume, rank, lastsale, name)
aug_last_day_rank <- last_day_rank |>
inner_join(info_of_those_that_passed_all_filters, by = "symbol") |>
arrange(desc(last_rank)) |>
select(symbol, last_rank, marketCap, marketCap_, accum_pct_change, country, industry, sector, ipoyear, volume, lastsale, name) |>
mutate(rank = row_number()) |>
mutate(symbol = paste0("<a href='https://finance.yahoo.com/quote/", symbol, "' target='_blank'>", symbol, "</a>")) |>
select(symbol, last_rank, marketCap, marketCap_, accum_pct_change, country, industry, sector, ipoyear, volume, rank, lastsale, name)
datatable(
aug_acc_rank,
filter = 'none',
options = list(
pageLength = 10,
autoWidth = TRUE,
dom = 'lrtip',
initComplete = JS(
"function(settings) {",
" var api = this.api();",
" var header = $(api.table().header());",
" var filterRow = $('<tr>').appendTo(header);",
" var colIndices = { country: 5, industry: 6, sector: 7, ipoyear: 8 };",
" var industryToSector = {};",
" // First pass: build industry-to-sector map",
" var allData = api.rows().data();",
" allData.each(function(row) {",
" var industry = row[colIndices.industry];",
" var sector = row[colIndices.sector];",
" if (industry && sector) {",
" industryToSector[industry] = sector;",
" }",
" });",
" api.columns().every(function(index) {",
" var column = this;",
" var cell = $('<th>').appendTo(filterRow);",
" if ([colIndices.country, colIndices.industry, colIndices.sector, colIndices.ipoyear].includes(index)) {",
" var select = $('<select><option value=\"\"></option></select>')",
" .appendTo(cell)",
" .on('change', function() {",
" var val = $.fn.dataTable.util.escapeRegex($(this).val());",
" column.search(val ? '^' + val + '$' : '', true, false).draw();",
" if (index === colIndices.sector) {",
" var selectedSector = $(this).val();",
" var industrySelect = filterRow.find('th').eq(colIndices.industry).find('select');",
" industrySelect.val(''); // clear selection",
" industrySelect.empty().append('<option value=\"\"></option>');",
" var industries = Object.keys(industryToSector).filter(function(ind) {",
" return selectedSector === '' || industryToSector[ind] === selectedSector;",
" }).sort();",
" industries.forEach(function(ind) {",
" industrySelect.append('<option value=\"' + ind + '\">' + ind + '</option>');",
" });",
" // Also clear industry filter when sector is reset",
" var industryColumn = api.column(colIndices.industry);",
" industryColumn.search('', true, false).draw();",
" }",
" });",
" column.data().unique().sort().each(function(d) {",
" select.append('<option value=\"' + d + '\">' + d + '</option>');",
" });",
" } else if (index === 0) {",
" // Add text input for symbol search",
" $('<input type=\"text\" placeholder=\"Search symbol\" style=\"width: 100%;\">')",
" .appendTo(cell)",
" .on('keyup change', function() {",
" var val = $(this).val();",
" column.search(val).draw();",
" });",
" } else {",
" $(cell).html('');",
" }",
" });",
"}"
)
),
escape = FALSE,
rownames = FALSE,
class = "display nowrap"
)
datatable(
aug_sum_rank,
filter = 'none',
options = list(
pageLength = 10,
autoWidth = TRUE,
dom = 'lrtip',
initComplete = JS(
"function(settings) {",
" var api = this.api();",
" var header = $(api.table().header());",
" var filterRow = $('<tr>').appendTo(header);",
" var colIndices = { country: 5, industry: 6, sector: 7, ipoyear: 8 };",
" var industryToSector = {};",
" // First pass: build industry-to-sector map",
" var allData = api.rows().data();",
" allData.each(function(row) {",
" var industry = row[colIndices.industry];",
" var sector = row[colIndices.sector];",
" if (industry && sector) {",
" industryToSector[industry] = sector;",
" }",
" });",
" api.columns().every(function(index) {",
" var column = this;",
" var cell = $('<th>').appendTo(filterRow);",
" if ([colIndices.country, colIndices.industry, colIndices.sector, colIndices.ipoyear].includes(index)) {",
" var select = $('<select><option value=\"\"></option></select>')",
" .appendTo(cell)",
" .on('change', function() {",
" var val = $.fn.dataTable.util.escapeRegex($(this).val());",
" column.search(val ? '^' + val + '$' : '', true, false).draw();",
" if (index === colIndices.sector) {",
" var selectedSector = $(this).val();",
" var industrySelect = filterRow.find('th').eq(colIndices.industry).find('select');",
" industrySelect.val(''); // clear selection",
" industrySelect.empty().append('<option value=\"\"></option>');",
" var industries = Object.keys(industryToSector).filter(function(ind) {",
" return selectedSector === '' || industryToSector[ind] === selectedSector;",
" }).sort();",
" industries.forEach(function(ind) {",
" industrySelect.append('<option value=\"' + ind + '\">' + ind + '</option>');",
" });",
" // Also clear industry filter when sector is reset",
" var industryColumn = api.column(colIndices.industry);",
" industryColumn.search('', true, false).draw();",
" }",
" });",
" column.data().unique().sort().each(function(d) {",
" select.append('<option value=\"' + d + '\">' + d + '</option>');",
" });",
" } else if (index === 0) {",
" // Add text input for symbol search",
" $('<input type=\"text\" placeholder=\"Search symbol\" style=\"width: 100%;\">')",
" .appendTo(cell)",
" .on('keyup change', function() {",
" var val = $(this).val();",
" column.search(val).draw();",
" });",
" } else {",
" $(cell).html('');",
" }",
" });",
"}"
)
),
escape = FALSE,
rownames = FALSE,
class = "display nowrap"
)
datatable(
aug_last_day_rank,
filter = 'none',
options = list(
pageLength = 10,
autoWidth = TRUE,
dom = 'lrtip',
initComplete = JS(
"function(settings) {",
" var api = this.api();",
" var header = $(api.table().header());",
" var filterRow = $('<tr>').appendTo(header);",
" var colIndices = { country: 5, industry: 6, sector: 7, ipoyear: 8 };",
" var industryToSector = {};",
" // First pass: build industry-to-sector map",
" var allData = api.rows().data();",
" allData.each(function(row) {",
" var industry = row[colIndices.industry];",
" var sector = row[colIndices.sector];",
" if (industry && sector) {",
" industryToSector[industry] = sector;",
" }",
" });",
" api.columns().every(function(index) {",
" var column = this;",
" var cell = $('<th>').appendTo(filterRow);",
" if ([colIndices.country, colIndices.industry, colIndices.sector, colIndices.ipoyear].includes(index)) {",
" var select = $('<select><option value=\"\"></option></select>')",
" .appendTo(cell)",
" .on('change', function() {",
" var val = $.fn.dataTable.util.escapeRegex($(this).val());",
" column.search(val ? '^' + val + '$' : '', true, false).draw();",
" if (index === colIndices.sector) {",
" var selectedSector = $(this).val();",
" var industrySelect = filterRow.find('th').eq(colIndices.industry).find('select');",
" industrySelect.val(''); // clear selection",
" industrySelect.empty().append('<option value=\"\"></option>');",
" var industries = Object.keys(industryToSector).filter(function(ind) {",
" return selectedSector === '' || industryToSector[ind] === selectedSector;",
" }).sort();",
" industries.forEach(function(ind) {",
" industrySelect.append('<option value=\"' + ind + '\">' + ind + '</option>');",
" });",
" // Also clear industry filter when sector is reset",
" var industryColumn = api.column(colIndices.industry);",
" industryColumn.search('', true, false).draw();",
" }",
" });",
" column.data().unique().sort().each(function(d) {",
" select.append('<option value=\"' + d + '\">' + d + '</option>');",
" });",
" } else if (index === 0) {",
" // Add text input for symbol search",
" $('<input type=\"text\" placeholder=\"Search symbol\" style=\"width: 100%;\">')",
" .appendTo(cell)",
" .on('keyup change', function() {",
" var val = $(this).val();",
" column.search(val).draw();",
" });",
" } else {",
" $(cell).html('');",
" }",
" });",
"}"
)
),
escape = FALSE,
rownames = FALSE,
class = "display nowrap"
)
datatable(
reason_for_those_that_did_not_pass_all_filters,
filter = "top", # Show filter boxes on top of the columns
options = list(
pageLength = 5,
autoWidth = TRUE,
columnDefs = list(
list(targets = 1, searchable = TRUE) # Disable search on the 'comment' column
)
)
)
## [1] "This was executed on: 2025-07-01 18:00:24.488087"
slackr_msg(
text = "New update at <https://leninrafaelrierasegura.github.io/Alpaca/info.html>",
channel = "#ranks"
)
We used R version 4.5.0 (R Core Team 2025) and the following R packages: DT v. 0.33 (Xie, Cheng, and Tan 2024), here v. 1.0.1 (Müller 2020), htmltools v. 0.5.8.1 (Cheng et al. 2024), knitr v. 1.50 (Xie 2014, 2015, 2025), renv v. 1.0.7 (Ushey and Wickham 2024), rmarkdown v. 2.29 (Xie, Allaire, and Grolemund 2018; Xie, Dervieux, and Riederer 2020; Allaire et al. 2024), slackr v. 3.3.1 (Rudis et al. 2023), tidyverse v. 2.0.0 (Wickham et al. 2019), xaringanExtra v. 0.8.0 (Aden-Buie and Warkentin 2024), zoo v. 1.8.14 (Zeileis and Grothendieck 2005).