-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathplugins.py
134 lines (106 loc) · 5.54 KB
/
plugins.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
import pandas as pd
import re
import os
import datetime
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from util import download_from_url, save_dataframe, generate_color_palette, draw_bar_graph
# Set the colour palette for the graph here, a minimum 2 colours is recommended
diverging_palette = ['#2c3071', "#225182", "#1f708a", "#42928e", '#5ea990', '#a1cb90']
qualitative_palette = ['#0fb5ae', '#4046ca', '#f68511', '#de3d82', '#7e84fa', '#72e06a',
'#147af3', '#7326d3', '#e8c600', '#cb5d00', '#008f5d', '#bce931']
def graph_plugins(configuration):
if configuration['download'] or configuration['provided_file']:
if configuration['download']:
data = download_from_url(
"https://raw.githubusercontent.com/obsidianmd/obsidian-releases/master/community-plugin-stats.json",
"raw-data/plugins.json" if configuration['save_data'] else "").decode("utf-8")
timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
else:
with open(configuration['provided_file'], 'r') as f:
data = f.read()
# Find if filename contains timestamp
if re.search(r'\d{4}.?\d{2}.?\d{2}', configuration['provided_file']):
timestamp = re.search(r'\d{4}.?\d{2}.?\d{2}', configuration['provided_file']).group(0) + 'T00-00-00'
else:
timestamp = datetime.datetime.fromtimestamp(os.path.getmtime(configuration['provided_file'])).strftime("%Y-%m-%dT%H-%M-%S")
# Parse json file with pandas
df = pd.read_json(data)
df = df.sum().to_frame().reset_index()
df.columns = ['name', 'downloads']
df['downloads'] = df['downloads'] / 2
df['downloads'] = df['downloads'].astype(int)
# To csv with timestamp of current date
if configuration['save_data']:
save_dataframe(df, "processed-data/plugins.csv", timestamp)
else:
# Get last fetched plugins.json file in the raw folder
plugin_files = [f for f in os.listdir("processed-data") if re.match(r"plugins_\d{4}.*\.csv", f)]
plugin_files.sort()
if not plugin_files:
print("No plugins.json file found in processed data folder")
return
else:
with open(os.path.join("processed-data", plugin_files[-1]), "r") as file:
df = pd.read_csv(file)
if configuration["chronological"] or configuration["sorted"]:
palette = generate_color_palette(diverging_palette, len(df))
df['age'] = df.index
df['color'] = palette
if configuration["chronological"]:
draw_bar_graph(df, palette, legend=False, logarithm=configuration["logarithm"])
if configuration["sorted"]:
draw_bar_graph(df.sort_values(by='downloads', ascending=False).reset_index(), palette, logarithm=configuration["logarithm"])
if configuration["difference"]:
# Get last fetched plugins.json file in the raw folder
plugin_files = [f for f in os.listdir("processed-data") if re.match(r"plugins_\d{4}.*\.csv", f)]
plugin_files.sort()
current_day = plugin_files[-1].split("_")[1].split("T")[0]
plugin_files = plugin_files[:-1]
# Select only one file for each day, the latest one
grouped_by_day = {}
for file in plugin_files:
date = file.split("_")[1].split("T")[0]
if date not in grouped_by_day:
grouped_by_day[date] = [file]
else:
grouped_by_day[date].append(file)
plugin_files = [grouped_by_day[k][-1] for k in grouped_by_day.keys()]
days = list(grouped_by_day.keys())
for idx, file in enumerate(plugin_files):
diff_df = pd.read_csv(os.path.join("processed-data", file))
# Get difference of diff_df and df on same name
df = df.merge(diff_df, on='name', how='outer', suffixes=('', f'_{days[idx]}'))
# df[days[idx]] = df['downloads' if idx == 0 else days[idx - 1]] - diff_df['downloads']
intervals = []
for idx, day in enumerate(days[:-1]):
interval = f"[{day}, {days[idx + 1]}]"
df[interval] = df[f'downloads_{days[idx + 1]}'] - df[f'downloads_{day}']
intervals.append(interval)
interval = f"[{days[-1]}, {current_day}]"
df[interval] = df['downloads'] - df[f'downloads_{days[-1]}']
intervals.append(interval)
if configuration['save_data']:
df.to_csv("processed-data/plugins_diff.csv", index=False)
plt.figure(figsize=(20, 6))
if configuration["logarithm"]:
plt.yscale('log')
plt.xlim(-0.5, len(df) - .5)
plt.gca().spines['top'].set_visible(False)
plt.gca().spines['right'].set_visible(False)
# Draw graph for every interval
for idx, interval in enumerate(intervals):
plt.bar(df.index, df[interval].values, 1, color=qualitative_palette[idx % len(qualitative_palette)], label=interval, bottom=sum(df[intervals[:idx-1]]) if idx else None)
# Set x-axis labels with rotation
plt.xticks(df.index[::20], df.index.values[::20], rotation=45)
patches = [mpatches.Patch(color=qualitative_palette[i], label=intervals[i]) for i in range(len(intervals))]
plt.legend(handles=patches, loc='upper left')
plt.show()
if __name__ == '__main__':
graph_plugins({
'chronological': False,
'normalize': False,
'sorted': True,
'download': False,
'provided_file': None,
})