initial commit

Author: nickmckay

crossResourceWorklowScraper.Rproj

@@ -0,0 +1,13 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX

scrapeGithub.Rmd

@@ -0,0 +1,130 @@
+---
+title: "scrapeGithub"
+output: html_document
+---
+### An individual scripts a tool to link resources
+
+In this case a user has written a script to link a data resource, and the R package written for that resource, to resources in GitHub.  The intention here is to survey ways in which the data resource is being used in analytic workflows.
+
+```{r ropensci_links, echo=TRUE, warning=FALSE, message=FALSE, results='hide'}
+library(magrittr)
+library(tidyverse)
+
+ropensci_registry <- jsonlite::fromJSON("https://raw.githubusercontent.com/ropensci/roregistry/master/registry.json")
+
+packages <- ropensci_registry$packages
+
+dd <- filter(packages,ropensci_category == "data-access")
+all.ddcat <- unique(unlist(dd$category))
+
+to.use <- c("biology","archeology","maps","ecology","chemistry","conservation",
+            "geospatial","environmental","waether","climate","biodiversity",
+            "animals","plants","water","hydrology","noaa","NOAA","ecosystem",
+            "remote-sensing")
+
+dd.touse <- filter(dd,category %in% to.use)
+gh_token <- scan('gh.token', what = 'character')
+
+if('all_github.rds' %in% list.files('data')) {
+  test_pks <- readRDS('data/all_github.rds')
+} else {
+  test_pks <- list()  
+}
+
+script_home <- 'https://github.com/throughput-ec/throughputdb/blob/master/populate/case_study.Rmd'
+
+for (i in 1:length(packages)) { 
+  
+  # I had to serialize this to avoid getting caught by GitHub's abuse detection.
+  
+  x <- packages[i]
+  
+  
+  if (!length(test_pks) >= i) {
+    Sys.sleep(5) # This is probably longer than it needs to be. . . 
+
+    repos <- gh::gh(paste0('/search/code?q=library(',x,
+                           ')+in:file+language:R+extension:R+extension:Rmd'), 
+                     .token = gh_token)
+       
+    annotation_text <- paste0("The GitHub repository uses the package ",
+                              x, " in a `library()` or `require()` call.")
+    
+    repo_list <- unique(sapply(repos$items, function(x)x$repository$html_url))
+    
+    target_list <- lapply(repo_list,
+                          function(y) { 
+                            if(length(repo_list) > 0) {
+                              return(object(value = y, type = 'URL')) 
+                              } else { return(NULL) }
+                            }
+                          )
+    
+    if (length(target_list) > 0) {
+    
+      test_pks[[i]] <- list(
+        target = target_list,
+        body = list(object(value = paste0('http://github.com/ropensci/',x), 
+                                     type = 'URL'),
+                    object( type = "annotationText",
+                           value = annotation_text),
+                    object(type = 'URL',
+                          value = script_home)),
+        generator = creator(identifier = '0000-0002-2700-4605',
+                                    PropertyID = 'orcid',
+                                      lastName = 'Goring',
+                                     firstName = 'Simon'),
+                body_rel = object(type = 'URL',
+                                 value = 'https://ropensci.org/'),
+                source = object(type = 'URL',
+                                value = 'http://github.com'))
+    } else {
+      test_pks[[i]] <- list()
+    }
+  }
+  
+  if (length(test_pks[[i]]) > 0) {
+    
+    link_record(con,
+                target = test_pks[[i]]$target,
+                  body = test_pks[[i]]$body,
+             generator = test_pks[[i]]$generator,
+              body_rel = test_pks[[i]]$body_rel,
+                source = test_pks[[i]]$source)
+  }
+  
+  saveRDS(object = test_pks, file = 'data/all_github.rds')
+  cat(i, '\n')
+  
+}
+
+```
+
+<div class='box curveshdw' align='center'>
+
+```{r ropensci_graph, echo=FALSE, fig.align = 'center', fig.width = 4, fig.caption = 'rOpenSci package use within the GitHub environment.'}
+
+nodes <- cypher(con, "MATCH (n1)-[r1]-(n2)
+                      RETURN DISTINCT ID(n1) AS id, LABELS(n1) AS type, '12' AS size LIMIT 300")
+
+edges <- cypher(con, "MATCH (n1)-[r1]-(n2)
+                      RETURN ID(n1) AS source, ID(n2) AS target, type(r1) AS type LIMIT 300")
+
+fd_data <- data.frame(source = (0:(nrow(nodes)-1))[match(edges$source, nodes$id)],
+                      target = (0:(nrow(nodes)-1))[match(edges$target, nodes$id)],
+                      rel    = edges$type)
+
+forceNetwork(Links = fd_data, 
+             Nodes = nodes, 
+             Source = 'source', 
+             Target = 'target', 
+             Value = 'rel',
+             NodeID = 'type',
+             Group = 'type',
+             Nodesize = 'size',
+             height = 600,
+             width = 600,
+             opacity = 1,
+             colourScale = JS("d3.scaleOrdinal(d3.schemeCategory10);"))
+
+```