Merge pull request #17 from datatogether/sprint_prep

Sprint prep

Author: patcon

.circleci/config.yml

@@ -35,12 +35,15 @@ jobs:
           command: go-wrapper download && go-wrapper install && go get -v github.com/jstemmer/go-junit-report
       - run: 
           name: Run tests
-          command: go test -v -race ./... | tee /tmp/test-reports/datatogether/original.txt ; test ${PIPESTATUS[0]} -eq 0
+          command: go test -v -race --coverprofile=coverage.txt -covermode=atomic | tee /tmp/test-reports/datatogether/original.txt ; test ${PIPESTATUS[0]} -eq 0
       - run: 
           name: Convert test output to junit-style xml
           command: cat /tmp/test-reports/datatogether/original.txt | go-junit-report > /tmp/test-reports/datatogether/junit.xml
       - store_test_results:
           path: /tmp/test-reports/datatogether/junit.xml
+      - run:
+          name: Publish coverage info to codecov.io
+          command: bash <(curl -s https://codecov.io/bash)
       - setup_remote_docker
       - run:
           name: Install Docker client

.github/CONTRIBUTING.md

@@ -0,0 +1,3 @@
+# Contributing Guidelines
+
+We love improvements to our tools! Take a moment to check out our organization-wide [Contributing Guidelines](https://github.com/datatogether/datatogether/blob/master/CONTRIBUTING.md) and [Code of Conduct](https://github.com/datatogether/datatogether/blob/master/CONDUCT.md).

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,12 @@
+Hey there, thank you for submitting an issue!
+
+We are trying to keep issues for feature requests and bug reports. Please
+complete the following checklist before creating a new one:
+
+- [ ] Is this a **bug report** (if so, is it something you can **debug and fix**?
+Send a pull request!)
+- [ ] feature request
+- [ ] support request => Please do not submit support requests here, ask your question
+on [Slack](https://archivers-slack.herokuapp.com/).
+
+---

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 gin-bin
 coverage
-coverage.out
+coverage.txt
 config.*.json
 *.env

Godeps/Godeps.json

@@ -0,0 +1,49 @@
+# Coverage
+
+[![GitHub](https://img.shields.io/badge/project-Data_Together-487b57.svg?style=flat-square)](http://github.com/datatogether)
+[![Slack](https://img.shields.io/badge/slack-Archivers-b44e88.svg?style=flat-square)](https://archivers-slack.herokuapp.com/)
+[![License](https://img.shields.io/github/license/datatogether/coverage.svg)](./LICENSE)
+[![Codecov](https://img.shields.io/codecov/c/github/datatogether/coverage.svg?style=flat-square)](https://codecov.io/gh/datatogether/coverage)
+
+Visualization to display "archival coverage," starting with epa.gov. This takes a list of urls and associated archiving information, and turns that into a tree of url paths with associated coverage information.
+
+The output is cached in `cache.json`, because this is a large file, we provide incremental pieces of the cached tree as a web server. To dynamically calculate coverage completion to can work with the `cache.json` file.
+
+## Current Coverage Sources
+
+Actual source datasets can be found in the `/repositories` directory. It currently includes the following:
+
+* Archivers 2
+* archivers.space
+* EDGI Nomination Tool Uncrawlables
+* The Internet Archive
+* Project Svalbard json-ld crawl
+
+## License & Copyright
+
+Copyright (C) 2017 Data Together
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU Affero General Public License as published by the Free Software
+Foundation, version 3.0.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.
+
+See the [`LICENSE`](./LICENSE) file for details.
+
+## Getting Involved
+
+We would love involvement from more people! If you notice any errors or would like to submit changes, please see our [Contributing Guidelines](./github/CONTRIBUTING.md).
+
+We use GitHub issues for [tracking bugs and feature requests](./issues) and Pull Requests (PRs) for [submitting changes](./pulls)
+
+## Installation
+
+The easiest way to get going is to use [docker-compose](https://docs.docker.com/compose/install/). Once you have that:
+
+TODO - finish installation instructions
+
+## Development
+
+Coming soon.

README.md

@@ -1,7 +1,7 @@
 package coverage
 
 import (
-	"github.com/datatogether/archive"
+	"github.com/datatogether/core"
 	"github.com/datatogether/coverage/repositories"
 	"github.com/datatogether/coverage/tree"
 
@@ -86,18 +86,18 @@ func WriteTreeCache(filename string, n *tree.Node) error {
 type CoverageGenerator struct {
 	// Root    url.Url
 	// Depth   int
-	Sources []*archive.Source
+	Sources []*core.Source
 	Repos   []repositories.CoverageRepository
 }
 
 // NewCoverageGenerator creates a CoverageGenerator with the default
 // properties
 func NewCoverageGenerator(repoIds []string, patterns []string) *CoverageGenerator {
-	var sources []*archive.Source
+	var sources []*core.Source
 	if patterns != nil {
-		sources := make([]*archive.Source, len(patterns))
+		sources := make([]*core.Source, len(patterns))
 		for i, pattern := range patterns {
-			sources[i] = &archive.Source{
+			sources[i] = &core.Source{
 				Url: pattern,
 			}
 		}

coverage/coverage.go

@@ -2,7 +2,7 @@ package coverage
 
 import (
 	"fmt"
-	"github.com/datatogether/archive"
+	"github.com/datatogether/core"
 	"github.com/datatogether/coverage/tree"
 	"net/url"
 	"strings"
@@ -72,9 +72,9 @@ func (p *CoverageSummaryParams) Validate() error {
 }
 
 func (CoverageRequests) Summary(p *CoverageSummaryParams, res *Summary) error {
-	sources := make([]*archive.Source, len(p.Patterns))
+	sources := make([]*core.Source, len(p.Patterns))
 	for i, p := range p.Patterns {
-		sources[i] = &archive.Source{Url: p}
+		sources[i] = &core.Source{Url: p}
 	}
 	summary, err := NewCoverageGenerator(p.RepoIds, p.Patterns).Summary()
 	if err != nil {

coverage/coverage_requests.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"database/sql"
-	"github.com/datatogether/archive"
+	"github.com/datatogether/core"
 	"github.com/datatogether/coverage/coverage"
 	"time"
 )
@@ -49,14 +49,14 @@ func calcSourceCoverage(db *sql.DB) error {
 	cvg := coverage.NewCoverageGenerator(nil, nil)
 	pageSize := 100
 
-	count, err := archive.CountSources(appDB)
+	count, err := core.CountSources(appDB)
 	if err != nil {
 		return err
 	}
 
 	numPages := count / pageSize
 	for page := 0; page <= numPages; page++ {
-		sources, err := archive.ListSources(store, pageSize, pageSize*page)
+		sources, err := core.ListSources(store, pageSize, pageSize*page)
 		if err != nil {
 			return err
 		}
@@ -68,7 +68,7 @@ func calcSourceCoverage(db *sql.DB) error {
 			}
 
 			if s.Stats == nil {
-				s.Stats = &archive.SourceStats{}
+				s.Stats = &core.SourceStats{}
 			}
 
 			if s.Stats.ArchivedUrlCount != summary.Archived {
@@ -88,14 +88,14 @@ func calcSourceCoverage(db *sql.DB) error {
 func calcPrimerSourceCoverage(db *sql.DB) error {
 	pageSize := 100
 
-	count, err := archive.CountPrimers(appDB)
+	count, err := core.CountPrimers(appDB)
 	if err != nil {
 		return err
 	}
 
 	numPages := int(count) / pageSize
 	for page := 0; page <= numPages; page++ {
-		primers, err := archive.ListPrimers(store, pageSize, pageSize*page)
+		primers, err := core.ListPrimers(store, pageSize, pageSize*page)
 		if err != nil {
 			return err
 		}
@@ -116,7 +116,7 @@ func calcPrimerSourceCoverage(db *sql.DB) error {
 			}
 
 			if p.Stats == nil {
-				p.Stats = &archive.PrimerStats{}
+				p.Stats = &core.PrimerStats{}
 			}
 
 			if p.Stats.SourcesUrlCount != urlCount || p.Stats.SourcesArchivedUrlCount != archivedCount {
@@ -133,7 +133,7 @@ func calcPrimerSourceCoverage(db *sql.DB) error {
 }
 
 // TODO - finish
-func calcPrimerCoverage(db *sql.DB, primers []*archive.Primer) error {
+func calcPrimerCoverage(db *sql.DB, primers []*core.Primer) error {
 	for _, primer := range primers {
 		if err := primer.ReadSubPrimers(db); err != nil {
 			return err