Update users channels and messages via API

internal/slacklog/api.go

@@ -0,0 +1,168 @@
+package slacklog
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+)
+
+func reverse(slice []interface{}) {
+	halfLen := len(slice) / 2
+	for i := 0; i < halfLen; i++ {
+		j := len(slice) - i - 1
+		slice[i], slice[j] = slice[j], slice[i]
+	}
+}
+
+func DownloadEntitiesToFile(slackToken string, apiMethod string, extraParams map[string]string, jsonKey string, isReverse bool, destJSONFilePath string) error {
+	results, err := getPaginatedEntites(slackToken, apiMethod, extraParams, jsonKey, isReverse)
+	if err != nil {
+		return err
+	}
+
+	if len(results) == 0 {
+		return nil
+	}
+
+	destDir := filepath.Dir(destJSONFilePath)
+	err = os.MkdirAll(destDir, 0777)
+	if err != nil {
+		return err
+	}
+
+	w, err := os.Create(destJSONFilePath)
+	if err != nil {
+		return err
+	}
+	defer w.Close()
+
+	encoder := json.NewEncoder(w)
+	encoder.SetEscapeHTML(false)
+	err = encoder.Encode(&results)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getPaginatedEntites(slackToken string, apiMethod string, extraParams map[string]string, jsonKey string, isReverse bool) ([]interface{}, error) {
+	// Slack API の pagination の仕様: https://api.slack.com/docs/pagination
+	var results []interface{}
+	var nextCursor string
+
+	pageNum := 1
+	for {
+		var json map[string]interface{}
+
+		log.Printf("GET %v page %d of %s ... ", extraParams, pageNum, apiMethod)
+		params := map[string]string{
+			"token":  slackToken,
+			"cursor": nextCursor,
+		}
+		for key, value := range extraParams {
+			params[key] = value
+		}
+		err := httpGetJSON("https://slack.com/api/"+apiMethod, params, &json)
+		if err != nil {
+			return nil, err
+		}
+
+		responseOK := true
+		okResponseValue, ok := json["ok"]
+		if ok {
+			responseOK, ok = okResponseValue.(bool)
+			if !ok {
+				responseOK = false
+			}
+		} else {
+			responseOK = false
+		}
+		if !responseOK {
+			errorMessage, ok := json["error"].(string)
+			if ok {
+				return nil, fmt.Errorf("Error response: %s", errorMessage)
+			}
+			return nil, fmt.Errorf("Error response: %v", json)
+		}
+
+		targetItem, ok := json[jsonKey]
+		if !ok {
+			return nil, fmt.Errorf("Key not found in response:" + jsonKey)
+		}
+		list, ok := targetItem.([]interface{})
+		if !ok {
+			return nil, fmt.Errorf("Unknown type of value:" + jsonKey)
+		}
+
+		log.Printf("fetched %s count: %d\n", jsonKey, len(list))
+
+		results = append(results, list...)
+
+		nextCursor = getNextCursor(json)
+		if nextCursor == "" {
+			break
+		}
+		pageNum += 1
+	}
+
+	if isReverse {
+		reverse(results)
+	}
+
+	return results, nil
+}
+
+func getNextCursor(json map[string]interface{}) string {
+	metadataValue, ok := json["response_metadata"]
+	if !ok {
+		return ""
+	}
+	metadata, ok := metadataValue.(map[string]interface{})
+	if !ok {
+		return ""
+	}
+	nextCursorValue, ok := metadata["next_cursor"]
+	if !ok {
+		return ""
+	}
+	nextCursor, ok := nextCursorValue.(string)
+	if !ok {
+		return ""
+	}
+	return nextCursor
+}
+
+func httpGetJSON(rawurl string, params map[string]string, dst interface{}) error {
+	url, err := url.Parse(rawurl)
+	if err != nil {
+		return err
+	}
+
+	values := url.Query()
+	for name, value := range params {
+		values.Set(name, value)
+	}
+	url.RawQuery = values.Encode()
+
+	resp, err := http.Get(url.String())
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("[%s]: %s", resp.Status, url.String())
+	}
+
+	err = json.NewDecoder(resp.Body).Decode(dst)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

subcmd/fetch_messages.go

@@ -0,0 +1,60 @@
+package subcmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/vim-jp/slacklog-generator/internal/slacklog"
+)
+
+func FetchMessages(args []string) error {
+	slackToken := os.Getenv("SLACK_TOKEN")
+	if slackToken == "" {
+		return fmt.Errorf("$SLACK_TOKEN required")
+	}
+
+	if len(args) < 2 {
+		fmt.Println("Usage: go run scripts/main.go update-messages {data-dir} {yyyy-mm-dd}")
+		return nil
+	}
+
+	dataDir := filepath.Clean(args[0])
+
+	loc, err := time.LoadLocation("Asia/Tokyo")
+	if err != nil {
+		return err
+	}
+
+	date, err := time.ParseInLocation("2006-01-02", args[1], loc)
+	if err != nil {
+		return err
+	}
+	dateEndTime := date.AddDate(0, 0, 1)
+
+	channelsJSONPath := filepath.Join(dataDir, "channels.json")
+	channelTable, err := slacklog.NewChannelTable(channelsJSONPath, []string{"*"})
+	if err != nil {
+		return err
+	}
+
+	for _, channel := range channelTable.Channels {
+		channelDir := filepath.Join(dataDir, channel.ID)
+
+		extraParams := map[string]string{
+			"channel": channel.ID,
+			"oldest":  strconv.FormatInt(date.Unix(), 10) + ".000000",
+			"latest":  strconv.FormatInt(dateEndTime.Unix(), 10) + ".000000",
+			"limit":   "200",
+		}
+		outFile := filepath.Join(channelDir, date.Format("2006-01-02")+".json")
+		err := slacklog.DownloadEntitiesToFile(slackToken, "conversations.history", extraParams, "messages", true, outFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

subcmd/subcmd.go

@@ -13,7 +13,10 @@ func Run() error {
     convert-exported-logs
     download-emoji
     download-files
-    generate-html`)
+    fetch-messages
+    generate-html
+    update-channel-list
+    update-user-list`)
 		return nil
 	}
 
@@ -26,8 +29,14 @@ func Run() error {
 		return DownloadEmoji(args)
 	case "download-files":
 		return DownloadFiles(args)
+	case "fetch-messages":
+		return FetchMessages(args)
 	case "generate-html":
 		return GenerateHTML(args)
+	case "update-channel-list":
+		return UpdateChannelList(args)
+	case "update-user-list":
+		return UpdateUserList(args)
 	}
 
 	return fmt.Errorf("unknown subcmd: %s", subCmdName)

subcmd/update_channel_list.go

@@ -0,0 +1,30 @@
+package subcmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/vim-jp/slacklog-generator/internal/slacklog"
+)
+
+func UpdateChannelList(args []string) error {
+	slackToken := os.Getenv("SLACK_TOKEN")
+	if slackToken == "" {
+		return fmt.Errorf("$SLACK_TOKEN required")
+	}
+
+	if len(args) < 1 {
+		fmt.Println("Usage: go run scripts/main.go update-channel-list {out-file}")
+		return nil
+	}
+
+	outFile := filepath.Clean(args[0])
+
+	err := slacklog.DownloadEntitiesToFile(slackToken, "conversations.list", nil, "channels", false, outFile)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

subcmd/update_user_list.go

@@ -0,0 +1,30 @@
+package subcmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/vim-jp/slacklog-generator/internal/slacklog"
+)
+
+func UpdateUserList(args []string) error {
+	slackToken := os.Getenv("SLACK_TOKEN")
+	if slackToken == "" {
+		return fmt.Errorf("$SLACK_TOKEN required")
+	}
+
+	if len(args) < 1 {
+		fmt.Println("Usage: go run scripts/main.go update-user-list {out-file}")
+		return nil
+	}
+
+	outFile := filepath.Clean(args[0])
+
+	err := slacklog.DownloadEntitiesToFile(slackToken, "users.list", nil, "members", false, outFile)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}