使用scrapy框架 获取网易财经沪深行情

Item

import scrapy
class MoneyItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    five_minute = scrapy.Field()  # 5分钟涨
    high = scrapy.Field()  # 最高
    hs = scrapy.Field()  # 换手率
    low = scrapy.Field()  # 最低
    mfsum = scrapy.Field()  # 每股收益
    name = scrapy.Field()  # 名称
    open = scrapy.Field()  # 今开
    pe = scrapy.Field()  # 市盈率
    percent = scrapy.Field()  # 涨跌幅
    price = scrapy.Field()  # 价格
    symbol = scrapy.Field()  # 代码
    updown = scrapy.Field()  # 涨跌额
    zf = scrapy.Field()  # 振幅
    no = scrapy.Field()  # 序号
    tcap = scrapy.Field()  # 总市值
    turnover = scrapy.Field()  # 成交额
    volume = scrapy.Field()  # 成交量
    yestclose = scrapy.Field()  # 昨收
    lb = scrapy.Field()  # 量比
    mcap = scrapy.Field()  # 流通市值
    mfratio2 = scrapy.Field()  # 净利润
    mfratio10 = scrapy.Field()  # 主营收
    wb = scrapy.Field()  # 委比
    query_time = scrapy.Field()  # 查询时间

为了方便将数据直接插入mysql，就用str改变了数据类型

# -*- coding: utf-8 -*-
import json
import scrapy
from money.items import MoneyItem
class GupiaoSpider(scrapy.Spider):
    name = 'gupiao'
    allowed_domains = ['quotes.money.com']    base_url = 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page={}&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=24&type=query'
    def start_requests(self):
        for page in range(160):
            url = self.base_url.format(page)
            yield scrapy.Request(url, callback=self.parse)
    def parse(self, response):
        content = json.loads(response.text)
        code_list = content['list']        item = MoneyItem()
        for code in code_list:
            item['query_time'] = content['time']            item['five_minute'] = str(code['FIVE_MINUTE'])
            item['high'] = str(code['HIGH'])
            if 'HS' in code.keys():
                item['hs'] = str(code['HS'])
            else:
                item['hs'] = ''
            item['low'] = str(code['LOW'])
            item['mfsum'] = str(code['MFSUM'])
            item['name'] = str(code['NAME'])
            item['open'] = str(code['OPEN'])
            if 'PE' in code.keys():
                item['pe'] = str(code['PE'])
            else:
                item['pe'] = ''
            item['percent'] = str(code['PERCENT'])
            item['price'] = str(code['PRICE'])
            item['symbol'] = str(code['SYMBOL'])
            item['updown'] = str(code['UPDOWN'])
            item['zf'] = str(code['ZF'])
            item['no'] = str(code['NO'])
            item['tcap'] = str(code['TCAP'])
            item['turnover'] = str(code['TURNOVER'])
            item['volume'] = str(code['VOLUME'])
            item['yestclose'] = str(code['YESTCLOSE'])
            if 'LB' in code.keys():
                item['lb'] = str(code['LB'])
            else:
                item['lb'] = ''
            if 'MCAP' in code.keys():
                item['mcap'] = str(code['MCAP'])
            else:
                item['mcap'] = ''
            item['mfratio2'] = str(code['MFRATIO']['MFRATIO2'])
            item['mfratio10'] = str(code['MFRATIO']['MFRATIO10'])
            item['wb'] = str(code['WB'])
            yield item

class MySQLPipeline(object):
    def __init__(self):
        self.db = pymysql.Connect(host=HOST, port=PORT, user=USER, password=PASSWORD, charset='utf8', db=DB)
        self.db_cur = self.db.cursor()
    def close_spider(self, spider):
        self.db.commit()
        self.db.close()
    def process_item(self, item, spider):
        self.insert(item)
        return item
    def insert(self, item):
        values = (item['query_time'], item['no'], item['symbol'], item['name'], item['price'], item['percent'], item['updown'],
                  item['five_minute'], item['open'], item['yestclose'], item['high'], item['low'],
                  item['volume'], item['turnover'], item['hs'], item['lb'], item['wb'], item['zf'],
                  item['pe'], item['mcap'], item['tcap'], item['mfsum'], item['mfratio2'], item['mfratio10']                  )
        sql = 'INSERT INTO stock VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
        self.db_cur.execute(sql, values)

import datetime
import time
from scrapy.cmdline import execute
def task():
    execute(['scrapy', 'crawl', 'gupiao'])
# 想几点更新,定时到几点
def run_time(h=15, m=0):
    while True:
        now = datetime.datetime.now()
        # print(now.hour, now.minute)
        if now.hour == h and now.minute == m:
            task()
        # 每隔60秒检测一次
        time.sleep(60)
if __name__ == '__main__':
    run_time()