Standardise stochastics

.github/workflows/R-CMD-check.yaml

@@ -1,14 +1,10 @@
-# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag.
-# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches:
-      - main
-      - master
+    branches: [main, master]
   pull_request:
-    branches:
-      - main
-      - master
+    branches: [main, master]
 
 name: R-CMD-check
 
@@ -22,61 +18,39 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
-          - {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
+          - {os: ubuntu-latest,   r: 'devel', http-user-agent: 'release'}
+          - {os: ubuntu-latest,   r: 'release'}
+
+    # Shorter timeout to prevent mac builders hanging for 6 hours!
+    timeout-minutes: 30
 
     env:
-      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
-      RSPM: ${{ matrix.config.rspm }}
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      R_KEEP_PKG_SOURCE: yes
+      R_CRAN_MIRROR: https://cloud.r-project.org
 
     steps:
-      - uses: actions/checkout@v2
-
-      - uses: r-lib/actions/setup-r@v1
-        with:
-          r-version: ${{ matrix.config.r }}
-
-      - uses: r-lib/actions/setup-pandoc@v1
+      - uses: actions/checkout@v4
 
-      - name: Query dependencies
-        run: |
-          install.packages('remotes')
-          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
-          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
-        shell: Rscript {0}
+      - uses: r-lib/actions/setup-pandoc@v2
 
-      - name: Cache R packages
-        if: runner.os != 'Windows'
-        uses: actions/cache@v2
+      - uses: r-lib/actions/setup-r@v2
         with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
+          r-version: ${{ matrix.config.r }}
+          http-user-agent: ${{ matrix.config.http-user-agent }}
+          use-public-rspm: true
 
-      - name: Install system dependencies
+      - name: Install system build tools
         if: runner.os == 'Linux'
         run: |
-          while read -r cmd
-          do
-            eval sudo $cmd
-          done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
+          sudo apt-get update
+          sudo apt-get install -y build-essential libcurl4-openssl-dev libssl-dev libxml2-dev
 
-      - name: Install dependencies
-        run: |
-          remotes::install_deps(dependencies = TRUE)
-          remotes::install_cran("rcmdcheck")
-        shell: Rscript {0}
-
-      - name: Check
-        env:
-          _R_CHECK_CRAN_INCOMING_REMOTE_: false
-        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check")
-        shell: Rscript {0}
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::rcmdcheck
+          needs: check
 
-      - name: Upload check results
-        if: failure()
-        uses: actions/upload-artifact@main
+      - uses: r-lib/actions/check-r-package@v2
         with:
-          name: ${{ runner.os }}-r${{ matrix.config.r }}-results
-          path: check
+          upload-snapshots: true

.github/workflows/pkgdown.yaml

@@ -1,48 +1,53 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches:
-      - main
-      - master
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+  release:
+    types: [published]
+  workflow_dispatch:
 
 name: pkgdown
 
 jobs:
   pkgdown:
-    runs-on: macOS-latest
+    runs-on: ubuntu-latest
+    # Only restrict concurrency for non-PR jobs
+    concurrency:
+      group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      R_CRAN_MIRROR: https://cloud.r-project.org
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - uses: r-lib/actions/setup-r@master
+      - uses: r-lib/actions/setup-pandoc@v2
 
-      - uses: r-lib/actions/setup-pandoc@master
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
 
-      - name: Query dependencies
+      - name: Install system build tools
+        if: runner.os == 'Linux'
         run: |
-          install.packages('remotes')
-          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
-          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
-        shell: Rscript {0}
+          sudo apt-get update
+          sudo apt-get install -y build-essential libcurl4-openssl-dev libssl-dev libxml2-dev
 
-      - name: Cache R packages
-        uses: actions/cache@v1
+      - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
+          extra-packages: any::pkgdown, local::.
+          needs: website
 
-      - name: Install dependencies
-        run: |
-          remotes::install_deps(dependencies = TRUE)
-          install.packages("pkgdown")
+      - name: Build site
+        run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
         shell: Rscript {0}
 
-      - name: Install package
-        run: R CMD INSTALL .
-
-      - name: Deploy package
-        run: |
-          git config --local user.email "actions@github.com"
-          git config --local user.name "GitHub Actions"
-          Rscript -e 'pkgdown::deploy_to_branch(new_process = FALSE)'
+      - name: Deploy to GitHub pages 🚀
+        if: github.event_name != 'pull_request'
+        uses: JamesIves/github-pages-deploy-action@v4.4.1
+        with:
+          clean: false
+          branch: gh-pages
+          folder: docs

.github/workflows/test-coverage.yaml

@@ -1,58 +1,57 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches:
-      - main
-      - master
+    branches: [main, master]
   pull_request:
-    branches:
-      - main
-      - master
+    branches: [main, master]
 
 name: test-coverage
 
 jobs:
   test-coverage:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
-      ORDERLYWEB_TEST_TOKEN: ${{ secrets.ORDERLYWEB_TEST_TOKEN }}
-    steps:
-      - uses: actions/checkout@v2
+      R_CRAN_MIRROR: https://cloud.r-project.org
 
-      - uses: r-lib/actions/setup-r@v1
+    steps:
+      - uses: actions/checkout@v4
 
-      - uses: r-lib/actions/setup-pandoc@v1
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
 
-      - name: Query dependencies
+      - name: Install system build tools
+        if: runner.os == 'Linux'
         run: |
-          install.packages('remotes')
-          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
-          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
-        shell: Rscript {0}
+          sudo apt-get update
+          sudo apt-get install -y build-essential libcurl4-openssl-dev libssl-dev libxml2-dev
 
-      - name: Cache R packages
-        uses: actions/cache@v2
+      - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
+          extra-packages: any::covr
+          needs: coverage
 
-      - name: Install system dependencies
-        if: runner.os == 'Linux'
+      - name: Test coverage
         run: |
-          while read -r cmd
-          do
-            eval sudo $cmd
-          done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
-          sudo apt-get install libcurl4-openssl-dev
+          covr::codecov(
+            quiet = FALSE,
+            clean = FALSE,
+            install_path = file.path(Sys.getenv("RUNNER_TEMP"), "package")
+          )
+        shell: Rscript {0}
 
-      - name: Install dependencies
+      - name: Show testthat output
+        if: always()
         run: |
-          install.packages(c("remotes"))
-          remotes::install_deps(dependencies = TRUE)
-          remotes::install_cran("covr")
-        shell: Rscript {0}
+          ## --------------------------------------------------------------------
+          find ${{ runner.temp }}/package -name 'testthat.Rout*' -exec cat '{}' \; || true
+        shell: bash
 
-      - name: Test coverage
-        run: covr::codecov()
-        shell: Rscript {0}
+      - name: Upload test results
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-test-failures
+          path: ${{ runner.temp }}/package

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: stoner
 Title: Support for Building VIMC Montagu Touchstones, using Dettl
-Version: 0.1.17
+Version: 0.1.18
 Authors@R:
     c(person("Wes", "Hinsley",role = c("aut", "cre", "cst", "dnc", "elg", "itr", "sng", "ard"),
            email = "w.hinsley@imperial.ac.uk"),
@@ -16,24 +16,25 @@ License: MIT + file LICENSE
 Encoding: UTF-8
 LazyData: true
 Imports:
+    arrow,
     DBI,
     data.table,
     jsonlite,
     lgr,
-    qs,
     dplyr,
     magrittr,
     prettyunits,
     readr,
+    rlang,
+    testthat,
     utils,
     withr
 Language: en-GB
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.3
 Roxygen: list(markdown = TRUE)
 Suggests:
     knitr,
     rcmdcheck,
     rmarkdown,
-    RPostgres,
-    testthat
+    RPostgres
 VignetteBuilder: knitr

NAMESPACE

@@ -4,16 +4,28 @@ export(stone_dump)
 export(stone_extract)
 export(stone_load)
 export(stone_stochastic_cert_verify)
+export(stone_stochastic_graph)
 export(stone_stochastic_process)
+export(stone_stochastic_standardise)
 export(stone_stochastic_upload)
 export(stone_test_extract)
 export(stone_test_transform)
 export(stone_transform)
 export(stoner_calculate_dalys)
 export(stoner_dalys_for_db)
-import(data.table)
+import(arrow)
+import(dplyr)
 import(readr)
+importFrom(arrow,write_parquet)
+importFrom(data.table,as.data.table)
+importFrom(data.table,rbindlist)
+importFrom(graphics,lines)
 importFrom(magrittr,"%>%")
+importFrom(rlang,":=")
+importFrom(stats,median)
+importFrom(stats,quantile)
+importFrom(testthat,expect_equal)
+importFrom(testthat,expect_true)
 importFrom(utils,capture.output)
 importFrom(utils,read.csv)
 importFrom(utils,write.csv)

R/dalys.R

@@ -8,6 +8,7 @@
 ##'
 ##' @export
 ##' @title Calculating DALYs for stochastic or central estimates
+##' @importFrom arrow write_parquet
 ##' @param con DBI connection to a Montagu database. Used for retrieving
 ##' demographic data for life expectancy.
 ##' @param touchstone The touchstone (including version); the demographic
@@ -232,7 +233,7 @@ stoner_life_table <- function(con, touchstone, year_min, year_max,
 ##' estimate set can be specified here, to extend that estimate set with
 ##' DALYs. Otherwise, leave as NULL to look up by the other four
 ##' fields.
-##' @param output_file If provided, then write the output CSV with the
+##' @param output_file If provided, then write an output pq file with the
 ##' additional DALYs field to this filename. The file will be ready to be
 ##' uploaded to Montagu.
 ##' @param life_table If provided, then re-use the life table provided by a
@@ -357,10 +358,10 @@ stoner_dalys_for_db <- function(con, dalys_params, modelling_group = NULL,
   data_dalys$data <- data_dalys$data[, c("disease", "year", "age", "country",
                                          "country_name", "cohort_size", cols)]
 
-  # Optionally write a CSV file out.
+  # Optionally write a parquet file out.
 
   if (!is.null(output_file)) {
-    qs::qsave(as.data.frame(data_dalys$data), output_file)
+    arrow::write_parquet(as.data.frame(data_dalys$data), output_file)
   }
 
   data_dalys

R/prune.R

@@ -1,4 +1,5 @@
 ##' @importFrom utils capture.output
+##' @importFrom testthat expect_true expect_equal
 
 extract_prune <- function(e, path, con) {
 
@@ -142,10 +143,10 @@ test_extract_prune <- function(e) {
   #
 
   bes <- e$burden_estimate_set
-  expect_true("id" %in% names(bes))
-  expect_true("responsibility" %in% names(bes))#
-  expect_equal(0, sum(is.na(bes$id)))
-  expect_equal(0, sum(is.na(bes$responsibility)))
+  testthat::expect_true("id" %in% names(bes))
+  testthat::expect_true("responsibility" %in% names(bes))#
+  testthat::expect_equal(0, sum(is.na(bes$id)))
+  testthat::expect_equal(0, sum(is.na(bes$responsibility)))
 
 }