Skip to content

Commit d2f7daf

Browse files
author
Shadowed
committed
Updated localization scraper
1 parent 3a0b552 commit d2f7daf

File tree

1 file changed

+189
-51
lines changed

1 file changed

+189
-51
lines changed

‎localcheck.lua

+189-51
Original file line numberDiff line numberDiff line change
@@ -1,92 +1,222 @@
1-
local ADDON_SLUG = string.match(io.open(".git/config"):read("*all"), "git%.wowace%.com:wow/(.-)/mainline%.git")
2-
--local CURSE_API_KEY = ""
3-
-- And you though I would let you steal my API key!
4-
dofile("../TestCode/api-key.lua")
5-
1+
-- If you want to hardcode toc or slug in and not enter it, do so here
2+
local ADDON_SLUG
63
local TOC_FILE
4+
-- Automatically identify the slug based on the .git or .svn configuration data, change the path if it's not in root, or set to false to disable
5+
local AUTO_IDENTIFY_SLUG = "./"
6+
-- Automatically find the TOC in the given path, set to false to disable
7+
local AUTO_FIND_TOC = "./"
8+
-- Only necessary if you aren't auto identify the slug, curseforge or wowace.
9+
local SITE_LOCATION = nil
10+
-- Personally I keep the api key in another file and just have this reference that to get it
11+
-- If you want to do this, create the file with CURSE_API_KEY = "<key>" in it and set the path here
12+
-- set this to nil and it will ask you for your API key
13+
local API_KEY_FILE = "../TestCode/api-key.lua"
14+
-- Patterns that should not be scrapped, case-insensitive
15+
-- Anything between the no-lib-strip is automatically ignored
16+
local FILE_BLACKLIST = {"^localization", "^lib"}
17+
-- 1 = English
18+
DEFAULT_LANGUAGE = "1"
19+
-- Removes phrases that are not found through this
20+
DELETE_UNIMPORTED = true
21+
22+
-- No more modifying!
23+
local OS_TYPE = os.getenv("HOME") and "linux" or "windows"
24+
25+
-- Mak sure we have LuaSockets
26+
local _, http = pcall(require, "socket.http")
27+
local _, ltn = pcall(require, "ltn12")
28+
if( not http ) then
29+
print("Failed to find socket.http, did you install LuaSockets?")
30+
elseif( not ltn ) then
31+
print("Failed to find ltn12, did you install LuaSockets?")
32+
end
33+
34+
35+
-- Figure out the API key
36+
if( API_KEY_FILE ) then
37+
local file = io.open(API_KEY_FILE)
38+
if( not file ) then
39+
print(string.format("It appears the API key file %s does not exist.", API_KEY_FILE))
40+
else
41+
file:close()
42+
dofile(API_KEY_FILE)
43+
44+
if( not CURSE_API_KEY ) then
45+
print("You did not define CURSE_API_KEY in your key file, make sure it does not have local next to it.")
46+
end
47+
end
48+
end
49+
750

8-
-- Check windows first
9-
local noLines = true
10-
for file in io.popen(string.format("dir /B \"./\"")):lines() do
11-
noLines = false
51+
if( not CURSE_API_KEY ) then
52+
while( not CURSE_API_KEY ) do
53+
io.stdout:write("Enter API key: ")
54+
CURSE_API_KEY = io.stdin:read("*line")
55+
CURSE_API_KEY = CURSE_API_KEY ~= "" and CURSE_API_KEY or nil
56+
end
57+
end
58+
59+
-- Attempt to automatically identify the addon slug
60+
if( AUTO_IDENTIFY_SLUG ) then
61+
local git = io.open(AUTO_IDENTIFY_SLUG .. ".git/config")
62+
local svn = io.open(AUTO_IDENTIFY_SLUG .. ".svn/entries")
1263

13-
if( string.match(file, "(.+)%.toc") ) then
14-
TOC_FILE = file
15-
break
64+
if( git ) then
65+
local contents = git:read("*all")
66+
git:close()
67+
68+
SITE_LOCATION, ADDON_SLUG = string.match(contents, "git%.(.-)%.com:wow/(.-)/mainline%.git")
69+
elseif( svn ) then
70+
local contents = svn:read("*all")
71+
svn:close()
72+
73+
SITE_LOCATION, ADDON_SLUG = string.match(contents, "svn%.(.-)%.com/wow/(.-)/mainline")
74+
end
75+
76+
if( not ADDON_SLUG ) then
77+
print("Failed to identify addon slug.")
78+
elseif( not SITE_LOCATION ) then
79+
print("Failed to identify site location.")
1680
end
1781
end
1882

19-
-- Now check OSX
20-
if( noLines ) then
21-
for file in io.popen(string.format("ls -1 \"./\"")):lines() do
22-
if( string.match(file, "(.+)%.toc") ) then
23-
TOC_FILE = file
24-
break
83+
if( not SITE_LOCATION ) then
84+
while( not SITE_LOCATION ) do
85+
io.stdout:write("Site location [wowace/curseforge]: ")
86+
SITE_LOCATION = io.stdin:read("*line")
87+
SITE_LOCATION = ( SITE_LOCATION == "wowace" or SITE_LOCATION == "curseforge" ) and SITE_LOCATION or nil
88+
end
89+
end
90+
91+
-- Manually ask for it
92+
if( not ADDON_SLUG ) then
93+
while( not ADDON_SLUG ) do
94+
io.stdout:write("Enter slug: ")
95+
ADDON_SLUG = io.stdin:read("*line")
96+
ADDON_SLUG = ADDON_SLUG ~= "" and ADDON_SLUG or nil
97+
end
98+
end
99+
100+
print(string.format("Using addon slug: %s", ADDON_SLUG))
101+
102+
-- Find the TOC now
103+
if( AUTO_FIND_TOC ) then
104+
local pipe = OS_TYPE == "windows" and io.popen(string.format("dir /B \"%s\"", AUTO_FIND_TOC)) or io.popen(string.format("ls -1 \"%s\"", AUTO_FIND_TOC))
105+
if( type(pipe) == "userdata" ) then
106+
for file in pipe:lines() do
107+
if( string.match(file, "(.+)%.toc") ) then
108+
TOC_FILE = file
109+
break
110+
end
25111
end
112+
113+
pipe:close()
114+
if( not TOC_FILE ) then print("Failed to auto detect toc file.") end
115+
else
116+
print("Failed to auto find toc, cannot run dir /B or ls -1")
26117
end
27118
end
28119

29120
if( not TOC_FILE ) then
30-
print("Failed to find toc file.")
31-
return
121+
while( not TOC_FILE ) do
122+
io.stdout:write("TOC path: ")
123+
TOC_FILE = io.stdin:read("*line")
124+
TOC_FILE = TOC_FILE ~= "" and TOC_FILE or nil
125+
if( TOC_FILE ) then
126+
local file = io.open(TOC_FILE)
127+
if( file ) then
128+
file:close()
129+
break
130+
else
131+
print(string.format("%s does not exist.", TOC_FILE))
132+
end
133+
end
134+
end
32135
end
33136

137+
print(string.format("Using TOC file %s", TOC_FILE))
138+
print("")
139+
34140
-- Parse through the TOC file so we know what to scan
35141
local ignore
36142
local localizedKeys = {}
37-
for text in io.lines(TOC_FILE) do
38-
if( string.match(text, "#@no%-lib%-strip@") ) then
143+
for line in io.lines(TOC_FILE) do
144+
line = string.gsub(line, "\r", "")
145+
146+
if( string.match(line, "#@no%-lib%-strip@") ) then
39147
ignore = true
40-
elseif( string.match(text, "#@end%-no%-lib%-strip@") ) then
148+
elseif( string.match(line, "#@end%-no%-lib%-strip@") ) then
41149
ignore = nil
42150
end
151+
152+
if( not ignore and string.match(line, "%.lua") ) then
153+
-- Make sure it's a valid file
154+
local blacklist
155+
for _, check in pairs(FILE_BLACKLIST) do
156+
if( string.match(string.lower(line), check) ) then
157+
blacklist = true
158+
break
159+
end
160+
end
43161

44-
if( not ignore and not string.match(text, "^localization") and string.match(text, "%.lua$") ) then
45-
local contents = io.open(text):read("*all")
162+
-- File checks out, scrap everything
163+
if( not blacklist ) then
164+
-- Fix slashes
165+
if( OS_TYPE == "linux" ) then
166+
line = string.gsub(line, "\\", "/")
167+
end
168+
169+
local keys = 0
170+
local contents = io.open(line):read("*all")
46171

47-
for match in string.gmatch(contents, "L%[\"(.-)%\"]") do
48-
localizedKeys[match] = true
172+
for match in string.gmatch(contents, "L%[\"(.-)%\"]") do
173+
if( not localizedKeys[match] ) then keys = keys + 1 end
174+
localizedKeys[match] = true
175+
end
176+
177+
print(string.format("%s (%d keys)", line, keys))
49178
end
50179
end
51180
end
52181

53-
-- Compile it into string form
182+
-- Compile all of the localization we found into string form
54183
local totalLocalizedKeys = 0
55-
local localization = "{"
184+
local localization = ""
56185
for key in pairs(localizedKeys) do
57-
localization = string.format("%s\n[\"%s\"] = \"%s\",", localization, key, key)
186+
localization = string.format("%s\nL[\"%s\"] = true", localization, key, key)
58187
totalLocalizedKeys = totalLocalizedKeys + 1
59188
end
60189

61-
localization = localization .. "\n}"
190+
if( totalLocalizedKeys == 0 ) then
191+
print("Warning, failed to find any localizations, perhaps you messed up a configuration variable?")
192+
return
193+
end
62194

63-
-- Send it all off to the localizer script
64-
local http = require("socket.http")
65-
local ltn = require("ltn12")
195+
print(string.format("Found %d keys total", totalLocalizedKeys))
66196

67197
local addonData = {
68-
["format"] = "lua_table",
69-
["language"] = "1",
70-
["delete_unimported"] = "y",
198+
["format"] = "lua_additive_table",
199+
["language"] = DEFAULT_LANGUAGE,
200+
["delete_unimported"] = DELETE_UNIMPORTED and "y" or "n",
71201
["text"] = localization,
72202
}
73203

74204
-- Send it off
75-
local boundary = "-------" .. os.time()
205+
local boundary = string.format("-------%s", os.time())
76206
local source = {}
77207
local body = ""
78208

79209
for key, data in pairs(addonData) do
80-
body = body .. "--" .. boundary .. "\r\n"
81-
body = body .. "Content-Disposition: form-data; name=\"" .. key .. "\"\r\n\r\n"
82-
body = body .. data .. "\r\n"
210+
body = string.format("%s--%s\r\n", body, boundary)
211+
body = string.format("%sContent-Disposition: form-data; name=\"%s\"\r\n\r\n", body, key)
212+
body = string.format("%s%s\r\n", body, data)
83213
end
84214

85-
body = body .. "--" .. boundary .. "\r\n"
215+
body = string.format("%s--%s\r\n", body, boundary)
86216

87217
http.request({
88218
method = "POST",
89-
url = string.format("http://www.wowace.com/addons/%s/localization/import/?api-key=%s", ADDON_SLUG, CURSE_API_KEY),
219+
url = string.format("http://www.%s.com/addons/%s/localization/import/?api-key=%s", SITE_LOCATION, ADDON_SLUG, CURSE_API_KEY),
90220
sink = ltn12.sink.table(source),
91221
source = ltn12.source.string(body),
92222
headers = {
@@ -95,14 +225,22 @@ http.request({
95225
},
96226
})
97227

98-
local VALID_LOCALIZATION = string.format("<a href=\"/addons/%s/localization/phrases/", ADDON_SLUG)
99-
--local file = io.open("test.txt", "w")
100-
for _, text in pairs(source) do
101-
if( string.match(text, "Redirecting%.%.%.") ) then
102-
print(string.format("Localization uploaded for %s, %s keys!", ADDON_SLUG, totalLocalizedKeys))
103-
return
228+
local contents = table.concat(source, "\n")
229+
local errors = {}
230+
for line in string.gmatch(contents, "(.-)\n") do
231+
local msg = string.match(line, "<span class=\"error\">(.-)</p>")
232+
if( msg ) then
233+
table.insert(errors, msg)
104234
end
105-
--file:write(text .. "\n")
106235
end
107236

108-
print(string.format("Failed to localize for %s :(", ADDON_SLUG))
237+
print("")
238+
239+
if( #(errors) == 0 ) then
240+
print(string.format("Updated localization for %s on %s!", ADDON_SLUG, SITE_LOCATION))
241+
else
242+
print("Localization failed:")
243+
for _, line in pairs(errors) do
244+
print(line)
245+
end
246+
end

0 commit comments

Comments
 (0)