From 3b4a61a9ee6d42f03c743ecd68bcc9afbc515fac Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Wed, 19 Mar 2025 14:33:47 +0100 Subject: [PATCH 1/7] fixes #76 --- R/readFitbit.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/readFitbit.R b/R/readFitbit.R index 5da0114..1a4434a 100644 --- a/R/readFitbit.R +++ b/R/readFitbit.R @@ -32,6 +32,7 @@ readFitbit = function(filename = NULL, desiredtz = "", if (dataType == "sleep") { epochSize = 30 # Put all data in data.frame + first_block_found = FALSE for (i in 1:length(D)) { tmp = D[[i]][15]$levels data = as.data.frame(data.table::rbindlist(tmp$data, fill = TRUE)) @@ -44,8 +45,9 @@ readFitbit = function(filename = NULL, desiredtz = "", if ("shortData" %in% names(tmp)) { shortData = data.table::rbindlist(tmp$shortData, fill = TRUE) shortData$dateTime = as.POSIXct(shortData$dateTime, format = "%Y-%m-%dT%H:%M:%S", tz = configtz) - if (i == 1) { + if (first_block_found == FALSE) { all_shortData = shortData + first_block_found = TRUE } else { all_shortData = rbind(all_shortData, shortData) } From bb480adc8ce0533115f055860f95729bd07758fe Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 21 Mar 2025 14:40:51 +0100 Subject: [PATCH 2/7] handle multiple json files of the same type when merging #76 --- R/mergeFitbitData.R | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/R/mergeFitbitData.R b/R/mergeFitbitData.R index 7163b5e..5cbb856 100644 --- a/R/mergeFitbitData.R +++ b/R/mergeFitbitData.R @@ -13,10 +13,33 @@ mergeFitbitData = function(filenames = NULL, desiredtz = "", configtz = NULL) { basename(filenames[cnt]), " and ", basename(filenames[cnt - 1])), call. = FALSE) } - - data = merge(data, D, by = "dateTime", all = TRUE) + # double names is possible when recording is split across json files + # in that case there may be multiple calories, steps and sleep files + doubleNames = colnames(D)[colnames(D) %in% colnames(data)] + new_times = which(D$dateTime %in% data$dateTime == FALSE) + double_times = which(D$dateTime %in% data$dateTime == TRUE) + if (length(new_times) > 0) { + data = merge(data, D[new_times, ], by = doubleNames, all = TRUE) + } + if (length(double_times) > 0) { + by_names = colnames(D)[colnames(D) %in% c("dateTime", "seconds")] + data2 = merge(data, D[double_times,], by = by_names, all.x = TRUE) + xcol = grep(pattern = "[.]x", x = colnames(data2)) + ycol = grep(pattern = "[.]y", x = colnames(data2)) + if (length(xcol) > 0 && length(ycol) > 0) { + # check whether new column has values that were missing before + replace_times = which(is.na(data2[, xcol]) & !is.na(data2[, ycol])) + if (length(replace_times) > 0) { + # replace previously missing values by new values + data2[replace_times, xcol] = data2[replace_times, ycol] + } + } + colnames(data2)[xcol] = gsub(pattern = "[.]x", replacement = "", x = colnames(data2)[xcol]) + data = data2[, -ycol] + } } cnt = cnt + 1 } + data = data[order(data$dateTime),] return(data) } \ No newline at end of file From b20cb8352751a50691be05352d66e705db4f8a3b Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 25 Mar 2025 11:02:23 +0100 Subject: [PATCH 3/7] extract light sensor data from Actiwatch if present --- R/readActiwatchCount.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/readActiwatchCount.R b/R/readActiwatchCount.R index 41d2ce6..0d47e0b 100644 --- a/R/readActiwatchCount.R +++ b/R/readActiwatchCount.R @@ -35,8 +35,8 @@ readActiwatchCount = function(filename = NULL, colnames(D)[grep(pattern = "activiteit|activity", x = colnames(D))] = "counts" colnames(D)[grep(pattern = "slapen|sleep", x = colnames(D))] = "sleep" colnames(D)[grep(pattern = "niet-om|wear|worn", x = colnames(D))] = "nonwear" - - D = D[, grep(pattern = "time|date|counts|sleep|nonwear|marker", x = colnames(D))] + colnames(D)[grep(pattern = "light", x = colnames(D))] = "light" + D = D[, grep(pattern = "time|date|counts|sleep|nonwear|marker|light", x = colnames(D))] timestamp_POSIX = as.POSIXct(x = paste(D$date[1:4], D$time[1:4], sep = " "), format = timeformat, tz = configtz) From f1c043aeaac7b449c8cb833611abaa2a7f556fa6 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 25 Mar 2025 17:02:23 +0100 Subject: [PATCH 4/7] Update NEWS.md --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 2e3cf1e..3525607 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# Changes in version 1.0.4 (release date:??-??-2025) + +- Fitbit: Fix bug preventing the loading of a sequence of json files. #76 + # Changes in version 1.0.3 (release date:07-03-2025) - Actiwatch: Extract marker button data from .AWD and .CSV files #72 From a4805e95c6be39918f21ea7ed275dd7c3140c67f Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 28 Mar 2025 14:06:41 +0100 Subject: [PATCH 5/7] correct extraction of Fitbit sleep data #76 --- R/readFitbit.R | 32 ++++++++++++-------------------- tests/testthat/test_readFitbit.R | 12 ++++++------ 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/R/readFitbit.R b/R/readFitbit.R index 1a4434a..106f6b9 100644 --- a/R/readFitbit.R +++ b/R/readFitbit.R @@ -54,29 +54,21 @@ readFitbit = function(filename = NULL, desiredtz = "", } } # Expand to full time series + all_data = all_data[order(all_data$dateTime),] D = as.data.frame(lapply(all_data, rep, all_data$seconds/epochSize)) - D$dateTime = seq(from = D$dateTime[1], length.out = nrow(D), by = epochSize) - D$seconds = epochSize - D = handleTimeGaps(D, epochSize) # Handle time gaps, if any - - S = as.data.frame(lapply(all_shortData, rep, all_shortData$seconds/30)) - S$dateTime = seq(from = S$dateTime[1], length.out = nrow(S), by = 30) - S$seconds = epochSize - - # merge in shortData (S) - matching_times = which(S$dateTime %in% D$dateTime == TRUE) - non_matching_times = which(S$dateTime %in% D$dateTime == FALSE) - if (length(matching_times) > 0) { - times_to_replace = S$dateTime[matching_times] - D[which(D$dateTime %in% times_to_replace), ] = S[matching_times,] - } - if (length(non_matching_times) > 0) { - D = rbind(D, S[non_matching_times,]) + all_shortData = all_shortData[order(all_shortData$dateTime),] + S = as.data.frame(lapply(all_shortData, rep, all_shortData$seconds/epochSize)) + D = rbind(D, S) + D = D[order(D$dateTime),] + D$seconds = epochSize + dup_indices = which(duplicated(D$dateTime)) + for (j in dup_indices) { + D$dateTime[j] = D$dateTime[j - 1] + epochSize } D = handleTimeGaps(D, epochSize) # Handle new time gaps, if any - - # Order time stamps - D = D[order(D$dateTime), ] + # wake overrules other classifications + dup_times = unique(D$dateTime[duplicated(D$dateTime)]) + D = D[-which(D$dateTime %in% dup_times & D$level != "wake"), ] colnames(D)[2] = "sleeplevel" } else if (dataType == "steps" || dataType == "calories") { data = as.data.frame(data.table::rbindlist(D, fill = TRUE)) diff --git a/tests/testthat/test_readFitbit.R b/tests/testthat/test_readFitbit.R index a2fdaec..d2623ef 100644 --- a/tests/testthat/test_readFitbit.R +++ b/tests/testthat/test_readFitbit.R @@ -4,12 +4,12 @@ test_that("Fitbit json is correctly read", { # Sleep file = system.file("testfiles/sleep-1995-06-23_Fitbit.json", package = "GGIRread") D = readFitbit(filename = file, desiredtz = "Europe/Amsterdam") - expect_equal(nrow(D), 695) + expect_equal(nrow(D), 47059) expect_equal(ncol(D), 3) - expect_equal(format(D$dateTime[1]), "1995-07-11 02:28:30") + expect_equal(format(D$dateTime[1]), "1995-06-24 22:47:30") TB = table(D$sleeplevel) expect_equal(names(TB), c("asleep", "awake", "deep", "light", "rem", "restless", "wake")) - expect_equal(as.numeric(TB), c(146, 2, 118, 283, 71, 10, 65)) + expect_equal(as.numeric(TB), c(146, 2, 118, 287, 67, 10, 65)) # Steps file = system.file("testfiles/steps-1995-06-23_Fitbit.json", package = "GGIRread") @@ -32,14 +32,14 @@ test_that("Timezones are correctly handled", { file = system.file("testfiles/sleep-1995-06-23_Fitbit.json", package = "GGIRread") # Configured and worn in same place D = readFitbit(filename = file, desiredtz = "Europe/Amsterdam") - expect_equal(format(D$dateTime[1]), "1995-07-11 02:28:30") + expect_equal(format(D$dateTime[1]), "1995-06-24 22:47:30") # Configured 1 hour earlier than timezone where device was worn D = readFitbit(filename = file, desiredtz = "Europe/London", configtz = "Europe/Amsterdam") - expect_equal(format(D$dateTime[1]), "1995-07-11 01:28:30") + expect_equal(format(D$dateTime[1]), "1995-06-24 21:47:30") # Configured 6 hours later than timezone where device was worn D = readFitbit(filename = file, desiredtz = "Europe/Amsterdam", configtz = "America/New_York") - expect_equal(format(D$dateTime[1]), "1995-07-11 08:28:30") + expect_equal(format(D$dateTime[1]), "1995-06-25 04:47:30") }) From f09c766496287f21bc122f62c6eadb094e8491f4 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 28 Mar 2025 16:26:15 +0100 Subject: [PATCH 6/7] correct and speed up merging time series fitbit #76 --- R/mergeFitbitData.R | 24 ++++++++++++++++-------- R/readFitbit.R | 15 +++++++-------- tests/testthat/test_readFitbit.R | 6 +++--- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/R/mergeFitbitData.R b/R/mergeFitbitData.R index 5cbb856..4fa4dbe 100644 --- a/R/mergeFitbitData.R +++ b/R/mergeFitbitData.R @@ -8,18 +8,18 @@ mergeFitbitData = function(filenames = NULL, desiredtz = "", configtz = NULL) { if (cnt == 1) { data = D } else { - if (length(intersect(x = data$dateTime, D$dateTime)) == 0) { - warning(paste0("Time series do not intersect for files ", - basename(filenames[cnt]), " and ", basename(filenames[cnt - 1])), - call. = FALSE) - } # double names is possible when recording is split across json files # in that case there may be multiple calories, steps and sleep files doubleNames = colnames(D)[colnames(D) %in% colnames(data)] new_times = which(D$dateTime %in% data$dateTime == FALSE) double_times = which(D$dateTime %in% data$dateTime == TRUE) if (length(new_times) > 0) { - data = merge(data, D[new_times, ], by = doubleNames, all = TRUE) + if (all(colnames(data) %in% colnames(D))) { + data = rbind(data, D[new_times, ]) + data = data[order(data$dateTime),] + } else { + data = merge(data, D[new_times, ], by = doubleNames, all = TRUE) + } } if (length(double_times) > 0) { by_names = colnames(D)[colnames(D) %in% c("dateTime", "seconds")] @@ -33,13 +33,21 @@ mergeFitbitData = function(filenames = NULL, desiredtz = "", configtz = NULL) { # replace previously missing values by new values data2[replace_times, xcol] = data2[replace_times, ycol] } + colnames(data2)[xcol] = gsub(pattern = "[.]x", replacement = "", x = colnames(data2)[xcol]) + data = data2[, -ycol] + } else { + data = data2 } - colnames(data2)[xcol] = gsub(pattern = "[.]x", replacement = "", x = colnames(data2)[xcol]) - data = data2[, -ycol] } } cnt = cnt + 1 } data = data[order(data$dateTime),] + + # fill gaps + timeRange = range(data$dateTime) + epochSize = min(diff(as.numeric(data$dateTime[1:pmin(10, nrow(data))]))) + timeFrame = data.frame(dateTime = seq( timeRange[1], timeRange[2], by = epochSize)) + data = merge(data, timeFrame, by = c("dateTime"), all.y = TRUE) return(data) } \ No newline at end of file diff --git a/R/readFitbit.R b/R/readFitbit.R index 106f6b9..6a48398 100644 --- a/R/readFitbit.R +++ b/R/readFitbit.R @@ -56,19 +56,18 @@ readFitbit = function(filename = NULL, desiredtz = "", # Expand to full time series all_data = all_data[order(all_data$dateTime),] D = as.data.frame(lapply(all_data, rep, all_data$seconds/epochSize)) - all_shortData = all_shortData[order(all_shortData$dateTime),] + D$index = unlist(mapply(seq, rep(0, nrow(all_data)), (all_data$seconds/epochSize) - 1)) + D$dateTime = D$dateTime + D$index * epochSize S = as.data.frame(lapply(all_shortData, rep, all_shortData$seconds/epochSize)) + S$index = unlist(mapply(seq, rep(0, nrow(all_shortData)), (all_shortData$seconds/epochSize) - 1)) + S$dateTime = S$dateTime + S$index * epochSize D = rbind(D, S) + D = D[, -which(colnames(D) %in% c("seconds", "index"))] D = D[order(D$dateTime),] - D$seconds = epochSize - dup_indices = which(duplicated(D$dateTime)) - for (j in dup_indices) { - D$dateTime[j] = D$dateTime[j - 1] + epochSize - } - D = handleTimeGaps(D, epochSize) # Handle new time gaps, if any - # wake overrules other classifications dup_times = unique(D$dateTime[duplicated(D$dateTime)]) + # wake overrules other classifications D = D[-which(D$dateTime %in% dup_times & D$level != "wake"), ] + D = D[!duplicated(D),] colnames(D)[2] = "sleeplevel" } else if (dataType == "steps" || dataType == "calories") { data = as.data.frame(data.table::rbindlist(D, fill = TRUE)) diff --git a/tests/testthat/test_readFitbit.R b/tests/testthat/test_readFitbit.R index d2623ef..90a08d6 100644 --- a/tests/testthat/test_readFitbit.R +++ b/tests/testthat/test_readFitbit.R @@ -4,12 +4,12 @@ test_that("Fitbit json is correctly read", { # Sleep file = system.file("testfiles/sleep-1995-06-23_Fitbit.json", package = "GGIRread") D = readFitbit(filename = file, desiredtz = "Europe/Amsterdam") - expect_equal(nrow(D), 47059) - expect_equal(ncol(D), 3) + expect_equal(nrow(D), 695) + expect_equal(ncol(D), 2) expect_equal(format(D$dateTime[1]), "1995-06-24 22:47:30") TB = table(D$sleeplevel) expect_equal(names(TB), c("asleep", "awake", "deep", "light", "rem", "restless", "wake")) - expect_equal(as.numeric(TB), c(146, 2, 118, 287, 67, 10, 65)) + expect_equal(as.numeric(TB), c(146, 2, 118, 285, 69, 10, 65)) # Steps file = system.file("testfiles/steps-1995-06-23_Fitbit.json", package = "GGIRread") From a718a681f2002f9b690fb84d3de5544ad4a2a609 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 28 Mar 2025 16:33:24 +0100 Subject: [PATCH 7/7] Update test_mergeFitbitData.R --- tests/testthat/test_mergeFitbitData.R | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/testthat/test_mergeFitbitData.R b/tests/testthat/test_mergeFitbitData.R index 6c27166..9ce06be 100644 --- a/tests/testthat/test_mergeFitbitData.R +++ b/tests/testthat/test_mergeFitbitData.R @@ -15,8 +15,10 @@ test_that("merging of PHB files goes correctly", { expect_equal(format(D$dateTime[1]), "1995-06-24 16:00:00") # apply function to merge the files - expect_warning(mergeFitbitData(filenames = c(file1, file2, file3), - desiredtz = "Europe/Amsterdam"), - regexp = "Time series*") + D2 = mergeFitbitData(filenames = c(file1, file2, file3), + desiredtz = "Europe/Amsterdam") + expect_true(all(colnames(D2) %in% c("dateTime", "steps", "calories", "sleeplevel"))) + expect_equal(nrow(D2), 47874) + rm(D2, D) }) \ No newline at end of file