Bin data by (x,y) and summarize

Question

These are the first 10 lines of a huge files I have: (Note that there is only one user in these 10 lines but I've got thousands of users)

dput(testd)
structure(list(user = c(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L
), otime = structure(c(10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L, 2L, 1L
), .Label = c("2010-10-12T19:56:49Z", "2010-10-13T03:57:23Z", 
"2010-10-13T16:41:35Z", "2010-10-13T20:05:43Z", "2010-10-13T23:31:51Z", 
"2010-10-14T00:21:47Z", "2010-10-14T18:25:51Z", "2010-10-16T03:48:54Z", 
"2010-10-16T06:02:04Z", "2010-10-17T01:48:53Z"), class = "factor"), 
    lat = c(39.747652, 39.891383, 39.891077, 39.750469, 39.752713, 
      39.752508, 39.7513, 39.758974, 39.827022, 39.749934),
    long = c(-104.99251, -105.070814, -105.068532, -104.999073, 
      -104.996337, -104.996637, -105.000121, -105.010853,
      -105.143191, -105.000017),
    locid = structure(c(5L, 4L, 9L, 6L, 1L, 2L, 8L, 3L, 10L, 7L),
      .Label = c("2ef143e12038c870038df53e0478cefc", 
      "424eb3dd143292f9e013efa00486c907", "6f5b96170b7744af3c7577fa35ed0b8f", 
      "7a0f88982aa015062b95e3b4843f9ca2", "88c46bf20db295831bd2d1718ad7e6f5", 
      "9848afcc62e500a01cf6fbf24b797732f8963683", "b3d356765cc8a4aa7ac5cd18caafd393", 
      "d268093afe06bd7d37d91c4d436e0c40d217b20a", "dd7cd3d264c2d063832db506fba8bf79", 
      "f6f52a75fd80e27e3770cd3a87054f27"), class = "factor"),
    dnt = structure(c(10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L, 2L, 1L),
      .Label = c("2010-10-12 19:56:49", 
      "2010-10-13 03:57:23", "2010-10-13 16:41:35", "2010-10-13 20:05:43", 
      "2010-10-13 23:31:51", "2010-10-14 00:21:47", "2010-10-14 18:25:51", 
      "2010-10-16 03:48:54", "2010-10-16 06:02:04", "2010-10-17 01:48:53"
    ), class = "factor"),
    x = c(-11674.6344476781, -11683.3414552141, 
      -11683.0877083915, -11675.3642199817, -11675.0599906624, 
      -11675.0933491404, -11675.4807522648, -11676.6740962175, 
      -11691.3894104198, -11675.4691879924),
    y = c(4419.73724843345, 4435.719406435, 4435.68538078744,
      4420.05048454181, 4420.3000059572, 4420.27721099723,
      4420.14288752585, 4420.99619739292, 4428.56278976123, 
      4419.99099525605),
    cellx = structure(c(1L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 2L, 1L),
      .Label = c("[-11682,-11672)", "[-11692,-11682)"
    ), class = "factor"),
    celly = structure(c(1L, 2L, 2L, 1L, 
      1L, 1L, 1L, 1L, 1L, 1L), .Label = c("[4419,4429)", "[4429,4439)"
    ), class = "factor"),
    cellxy = structure(c(1L, 3L, 3L, 1L, 
      1L, 1L, 1L, 1L, 2L, 1L), .Label = c("[-11682,-11672)[4419,4429)", 
      "[-11692,-11682)[4419,4429)", "[-11692,-11682)[4429,4439)"
    ), class = "factor")), .Names = c("user", "otime", "lat", 
"long", "locid", "dnt", "x", "y", "cellx", "celly", "cellxy"), class = "data.frame", row.names = c(NA, 
-10L))

A bit of explanation on what the data is to simplify understanding. The x and y are transformation of the lat and long coordinates. I have discretised the x,y locations into bins using cut. I want to get the most visited bin per user so I use ddply. As follows:

cells = ddply(testd, .(user, cellxy), summarise, length(cellxy))

Obtaining:

dput(cells)
structure(list(user = c(0, 0, 0), cellxy = structure(1:3, .Label = c("[-11682,-11672)[4419,4429)", 
"[-11692,-11682)[4419,4429)", "[-11692,-11682)[4429,4439)"), class = "factor"), 
    count = c(7L, 1L, 2L)), .Names = c("user", "cellxy", "count"
), row.names = c(NA, -3L), class = "data.frame")

Now what I want to do is calculate the average x,y from the first dataset for the most visited bin per user as obtained from the previous calculation. I have no idea how to do this efficiently and given that my dataset is really big I would appreciate some guidance. Thanks!

Overcharge

Hello, can you please elaborate on what you mean by "the average x,y..." How exactly are you computing this avg? – Musketry

Answer 1

Here is two stage approach. First, modified your original code of cells - for each combination of cellxy and user calculate mean x and y value.

 cells = ddply(testd, .(user, cellxy), summarise,
       cellcount=length(cellxy),meanx=mean(x),meany=mean(y))
 cells
  user                     cellxy cellcount     meanx    meany
1    0 [-11682,-11672)[4419,4429)         7 -11675.40 4420.214
2    0 [-11692,-11682)[4419,4429)         1 -11691.39 4428.563
3    0 [-11692,-11682)[4429,4439)         2 -11683.21 4435.702

Then use other call to ddply() to subset for each user cellxy with highest cellcount.

cells2 = ddply(cells,.(user),subset,cellcount==max(cellcount))
cells2
  user                     cellxy cellcount    meanx    meany
1    0 [-11682,-11672)[4419,4429)         7 -11675.4 4420.214

Clemen

I'm going to try it on a few different line and get bk. Thanks for the prompt reply. – Overcharge

Answer 2

since your data set is large, you might want to consider data.table, which not only will be blazing fast, it will also make the data mungling a bit easier.

Converting to a data table is straight forward:

    library (data.table)
    DT <- data.table(testd, by="user")

Then determining the most visited, by user, is just one line

    # Determining which is the most visited, by user 
    DT[, "MostVisited" := {counts <- table(cellxy); names(counts)[which(counts==max(counts))]}, by=user]

---

I'm not sure how specifically you want to calculate the average x, y relative to the MostVisited, but I'm sure that as well could be relatively straight forward with data.table.

    ## But perhaps something like this
    DT[, c("AvgX", "AvgY") := list(mean(x), mean(y)), by=list(user, MostVisited)]

Musketry

Thanks for the reply! I had not heard of data.table. Will try it out. I used Didzis's answer as it came in earlier though. Also what I mean by averages, was once I knew the most visited cell, I wanted to get the average x and y of all the (x,y) that were in that bin. Cheers! – Overcharge