hinna0818/ukbanalytica_v2/ukbsci-rap-extract

proj_root <- "/mnt/project"                          # RAP project storage
extract_dir <- file.path(proj_root, "extracts")
dir.create(extract_dir, showWarnings = FALSE, recursive = TRUE)

# Inside a RAP R session
install.packages("UKBAnalytica")   # or devtools::install_github("Hinna0818/UKBAnalytica")
library(UKBAnalytica)
 
# Confirm RAP signals and dx availability before extraction
env <- ukb_check_rap_env(require_rap = TRUE, require_dx = TRUE, verbose = TRUE)

env <- ukb_check_rap_env(
  require_rap = TRUE,
  require_dx  = TRUE,
  check_auth  = TRUE,
  verbose     = TRUE
)

# 1.1 Locate the .dataset file in the RAP project root
dataset <- rap_find_dataset()
#> e.g. "app99999_20231001154123.dataset"
 
# 1.2 List all available RAP field columns for the participant entity
fields_df <- rap_list_fields(dataset = dataset)
head(fields_df)
#>     field_name                     title
#> 1   participant.eid                Participant ID
#> 2   participant.p31_i0             Sex | Instance 0
#> ...
 
# 1.3 Optional: regex filter at list time (faster than later filtering)
bp <- rap_list_fields(dataset = dataset, pattern = "blood pressure")

# Mode A — pure RAP (no local dictionary)
meta <- ukb_metadata_setup(source = "rap", dataset = dataset)
 
# Mode B — offline (showcase TSVs only)
meta <- ukb_metadata_setup(
  source     = "files",
  data_dict  = "Data_Dictionary_Showcase.tsv",
  codings    = "Codings.tsv"
)
 
# Mode C — auto (combine RAP + showcase, the default)
meta <- ukb_metadata_setup(source = "auto",
                            data_dict = "Data_Dictionary_Showcase.tsv",
                            codings   = "Codings.tsv",
                            dataset   = dataset,
                            cache     = TRUE)

# Keyword search across title/description/category/field_name/rap_field_names/coding_id
hits <- ukb_search_fields(query = "systolic blood pressure",
                          metadata = meta, max_results = 20)
 
# Deep dive into one field
info <- ukb_field_info("4080", metadata = meta)         # by field_id
info <- ukb_field_info("p31_i0", metadata = meta)       # by RAP column
info <- ukb_field_info("age_at_recruitment", metadata = meta)  # by variable name
print(info)                                              # human summary
info$rap_fields                                          # all matching RAP columns
info$codings                                             # value labels (if coded)

get_field_info(field_id = 31, ukb_data_dict = "Data_Dictionary_Showcase.tsv")
# or, if internet is allowed on the RAP node:
get_field_info(field_id = 31, live = TRUE)

plan <- rap_plan_extract(
  field_id    = c(31, 21003, 4079, 4080, 21001),
  field_names = "participant.eid",         # exact RAP columns also accepted
  variables   = c("ethnic_background"),    # predefined sets from get_variable_info()
  dataset     = dataset,
  fields_df   = fields_df,                 # optional speed-up: skip re-listing
  manifest    = "/mnt/project/extracts/fields_manifest.csv"
)
 
plan$n_columns      # decides sync vs async
plan$fields         # exact column names that will be requested
plan$matched        # data.frame of resolved requests
plan$unmatched      # any unresolved requests (handle before running!)

manifest <- ukb_create_extraction_manifest(
  field_id  = c(31, 53, 21022, 21000, 21001),
  variables = c("sex", "age", "ethnicity", "bmi"),
  purpose   = "baseline cohort construction"
)
 
ukb_write_extraction_manifest(
  manifest,
  path   = "/mnt/project/<area>/02-extract/manifest.csv",
  format = "csv"
)

dt <- rap_extract_pheno(
  field_id  = plan$matched$field_id,            # or pass plan via field_names = plan$fields
  dataset   = dataset,
  output    = "/mnt/project/extracts/pheno_small.csv",  # path stays in RAP storage
  read      = TRUE,                              # load into R as data.table
  strip_entity_prefix = TRUE                     # "participant.p31" -> "p31"
)
str(dt)

job <- rap_submit_extract(
  field_id      = plan$matched$field_id,
  dataset       = dataset,
  file          = "pheno_full_baseline",        # output stem on RAP
  instance_type = NULL,                          # auto-pick based on n_columns
  priority      = "low"
)
job$job_id
#> "job-Gxxxxxxxxxx"

dx watch <job-id>
dx describe <job-id>

out_path <- file.path("/mnt/project", paste0(job$output, ".csv"))
dt <- data.table::fread(out_path)
# Do not print `head(dt)` or share row-level previews with the local agent.

# Rename UKB columns from cryptic IDs to readable names
dt <- ukb_decode_column_names(dt, metadata = meta,
                              style = "snake",        # "snake" | "title" | "field_id_title"
                              keep_instance = TRUE,
                              keep_array    = TRUE,
                              max_nchar     = 80)
 
# Recode coded values using the metadata codings table (keeps raw + adds label)
dt <- ukb_decode_values(dt, metadata = meta,
                        keep_raw     = TRUE,
                        suffix       = "_label",
                        missing_to_na = TRUE)
 
# Or do both at once:
dt <- ukb_decode(dt, metadata = meta,
                 decode_names = TRUE, decode_values = TRUE, keep_raw = TRUE)