Skip to content

Commit 4e082d0

Browse files
cxhuancxhuan
committed
提交代码
1 parent 428b73e commit 4e082d0

File tree

3 files changed

+209
-0
lines changed

3 files changed

+209
-0
lines changed

‎.DS_Store

0 Bytes
Binary file not shown.

‎xianhuan/.DS_Store

2 KB
Binary file not shown.

‎xianhuan/stockinfo/stock_day_info_163.py

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
#!/usr/bin/env python3
2+
# -*- coding: utf-8 -*-
3+
"""
4+
5+
网页财经每日行情排行榜，copy所有股票当日数据
6+
7+
@author: 闲欢
8+
"""
9+
10+
import math
11+
12+
import requests
13+
import json
14+
import traceback
15+
import pymysql
16+
17+
18+
19+
class StockDayInfo:
20+
21+
def __init__(self):
22+
self.ua_header = {"Connection": "keep-alive",
23+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36",
24+
"Host": "quotes.money.163.com",
25+
"Cookie": "vjuids=2453fea.1759e01b4ef.0.c69c7922974aa; _ntes_nnid=99f0981d725ac03af6da5eec0508354e,1604673713410; _ntes_nuid=99f0981d725ac03af6da5eec0508354e; _ntes_stock_recent_=1300033; ne_analysis_trace_id=1604846790608; s_n_f_l_n3=20f075946bacfe111604846790626; _antanalysis_s_id=1604933714338; vjlast=1604673713.1605426311.11; pgr_n_f_l_n3=20f075946bacfe1116055330765687243; vinfo_n_f_l_n3=20f075946bacfe11.1.0.1604846790623.0.1605533081941"
26+
}
27+
self.conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='root', db='east_money', charset='utf8')
28+
self.cur = self.conn.cursor()
29+
30+
def get_data(self, url):
31+
response = requests.get(url, headers=self.ua_header, verify=False)
32+
content = response.content.decode('unicode_escape')
33+
return content
34+
35+
def parse_data(self, data):
36+
result_obj = json.loads(data)
37+
38+
obj = {}
39+
obj['pagecount'] = result_obj['pagecount']
40+
obj['time'] = result_obj['time']
41+
obj['total'] = result_obj['total']
42+
list_str = result_obj['list']
43+
stock_list = []
44+
if list_str:
45+
data_list = list(list_str)
46+
for s in data_list:
47+
# print(s)
48+
stock = {}
49+
stock['query_code'] = s['CODE']
50+
stock['code'] = s['SYMBOL']
51+
stock['name'] = s['SNAME']
52+
if 'PRICE' in s.keys():
53+
stock['close_price'] = self.trans_float(s['PRICE'])
54+
else:
55+
stock['close_price'] = 0.00
56+
if 'HIGH' in s.keys():
57+
stock['top_price'] = self.trans_float(s['HIGH'])
58+
else:
59+
stock['top_price'] = 0.00
60+
if 'LOW' in s.keys():
61+
stock['low_price'] = self.trans_float(s['LOW'])
62+
else:
63+
stock['low_price'] = 0.00
64+
if 'OPEN' in s.keys():
65+
stock['open_price'] = self.trans_float(s['OPEN'])
66+
else:
67+
stock['open_price'] = 0.00
68+
if 'YESTCLOSE' in s.keys():
69+
stock['last_price'] = self.trans_float(s['YESTCLOSE'])
70+
else:
71+
stock['last_price'] = 0.00
72+
if 'UPDOWN' in s.keys():
73+
stock['add_point'] = self.trans_float(s['UPDOWN'])
74+
else:
75+
stock['add_point'] = 0.00
76+
if 'PERCENT' in s.keys():
77+
stock['add_percent'] = self.trans_float(s['PERCENT'])
78+
else:
79+
stock['add_percent'] = 0.00
80+
if 'HS' in s.keys():
81+
stock['exchange_rate'] = self.trans_float(s['HS'])
82+
else:
83+
stock['exchange_rate'] = 0.00
84+
if 'VOLUME' in s.keys():
85+
stock['volumn'] = self.trans_float(s['VOLUME'])
86+
else:
87+
stock['volumn'] = 0.00
88+
if 'TURNOVER' in s.keys():
89+
stock['turnover'] = self.trans_float(s['TURNOVER'])
90+
else:
91+
stock['turnover'] = 0.00
92+
if 'TCAP' in s.keys():
93+
stock['market_value'] = self.trans_float(s['TCAP'])
94+
else:
95+
stock['market_value'] = 0.00
96+
if 'MCAP' in s.keys():
97+
stock['flow_market_value'] = self.trans_float(s['MCAP'])
98+
else:
99+
stock['flow_market_value'] = 0.00
100+
stock_list.append(stock)
101+
102+
obj['stock'] = stock_list
103+
104+
return obj
105+
106+
@staticmethod
107+
def trans_float(s):
108+
try:
109+
return float(s)
110+
except Exception:
111+
return 0.00
112+
113+
def insert_db(self, obj_list, day):
114+
try:
115+
if len(obj_list):
116+
insert_attrs = ['day', 'query_code', 'code', 'name', 'close_price', 'top_price', 'low_price', 'open_price', 'last_price', 'add_point', 'add_percent', 'exchange_rate', 'volumn', 'turnover', 'market_value', 'flow_market_value']
117+
insert_tuple = []
118+
for obj in obj_list:
119+
insert_tuple.append((day,
120+
obj['query_code'],
121+
obj['code'],
122+
obj['name'],
123+
obj['close_price'],
124+
obj['top_price'],
125+
obj['low_price'],
126+
obj['open_price'],
127+
obj['last_price'],
128+
obj['add_point'],
129+
obj['add_percent'],
130+
obj['exchange_rate'],
131+
obj['volumn'],
132+
obj['turnover'],
133+
obj['market_value'],
134+
obj['flow_market_value']))
135+
values_sql = ['%s' for v in insert_attrs]
136+
attrs_sql = '('+','.join(insert_attrs)+')'
137+
values_sql = ' values('+','.join(values_sql)+')'
138+
sql = 'insert into %s' % 'stock_info'
139+
sql = sql + attrs_sql + values_sql
140+
try:
141+
print(sql)
142+
for i in range(0, len(insert_tuple), 20000):
143+
self.cur.executemany(sql, tuple(insert_tuple[i:i+20000]))
144+
self.conn.commit()
145+
except pymysql.Error as e:
146+
self.conn.rollback()
147+
error = 'insertMany executemany failed! ERROR (%s): %s' % (e.args[0], e.args[1])
148+
print(error)
149+
except Exception:
150+
#输出异常信息
151+
traceback.print_exc()
152+
153+
154+
def deal_json_invaild(self, data):
155+
data = data.replace("\n", "\\n").replace("\r", "\\r").replace("\n\r", "\\n\\r") \
156+
.replace("\r\n", "\\r\\n") \
157+
.replace("\t", "\\t")
158+
data = data.replace('":"', '&&GSRGSR&&')\
159+
.replace('":', "%%GSRGSR%%") \
160+
.replace('","', "$$GSRGSR$$")\
161+
.replace(',"', "~~GSRGSR~~") \
162+
.replace('{"', "@@GSRGSR@@") \
163+
.replace('"}', "**GSRGSR**")
164+
# print(data)
165+
166+
data = data.replace('"', r'\"') \
167+
.replace('&&GSRGSR&&', '":"')\
168+
.replace('%%GSRGSR%%', '":')\
169+
.replace('$$GSRGSR$$', '","')\
170+
.replace("~~GSRGSR~~", ',"')\
171+
.replace('@@GSRGSR@@', '{"')\
172+
.replace('**GSRGSR**', '"}')
173+
# print(data)
174+
return data
175+
176+
def deal(self):
177+
url = 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=0&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=5000&type=query'
178+
try:
179+
content = self.get_data(url)
180+
print(content)
181+
obj = self.parse_data(self.deal_json_invaild(content))
182+
time = obj['time']
183+
184+
data_list = obj['stock']
185+
if len(data_list):
186+
tmp_list = []
187+
if len(data_list) <= 100:
188+
tmp_list = data_list
189+
else:
190+
floor_num = math.floor(len(data_list)/100)
191+
for i in range(0, floor_num - 1):
192+
insert_list = data_list[100*i:100*(i+1) - 1]
193+
self.insert_db(insert_list, time[0:10])
194+
tmp_list = data_list[floor_num*100:len(data_list) - 1]
195+
196+
self.insert_db(tmp_list, time[0:10])
197+
except Exception as err:
198+
print(err)
199+
traceback.print_exc()
200+
pass
201+
202+
203+
if __name__ == "__main__":
204+
sdi = StockDayInfo()
205+
sdi.deal()
206+
# schedule.every().day.at('16:40').do(sdi.deal())
207+
# while True:
208+
# schedule.run_pending()
209+

0 commit comments

Comments
 (0)