利用爬虫技术能做到哪些很酷很有趣很有用的事情？ 第1页

       // 代码不全，只有主要的逻辑 // 用到的库如下：  var request = require('superagent'); var cheerio = require('cheerio'); var fs = require('fs');  // 首先是这样的一个接口，可以取到某个答案所有赞同的人数 // 每次取会返回10条数据，是编译好的HTML模版，还有下一组数据的地址 // 遍历这10条数据并取到所有人的ID即可 // config 是Cookie、Host、Referer等配置  var sourceLink = 'https://www.zhihu.com/answer/' + code + '/voters_profile';  function getVoterList(link, fn) { var next = ''; if (postListLength && !sleepIng) { console.log('waiting'); sleepIng = true; return setTimeout(function () { sleepIng = false; sleep = 1; getVoterList(link, fn); }, 1000 * 60); } request.get(link) .set(config) .end(function (err, res) { if (err || !res.ok) { return console.log(err); } var result = JSON.parse(res.text), voterList = '', $;  if (result.paging && result.paging.next) { next = result.paging.next; }  if (result.payload && result.payload.length) { voterList = result.payload.join(''); $ = cheerio.load(voterList);  $('.zm-rich-follow-btn').each(function () { var id = $(this).attr('data-id');  if (voterIdList.indexOf(id) === -1 && oldIdList.indexOf(id) === -1) { console.log('new id: ', id); voterIdList.push(id); } else { dupIdLen += 1; } }); }  if (next && dupIdLen < 20) { setTimeout(function () { getVoterList('https://www.zhihu.com' + next, fn); }, 3000); } else { dupIdLen = 0; fn(); } }); }  // 在爬取完该接口后，新的点赞人数会暂存在数组中，遍历该数组，并发送请求 // 如请求发送成功，将各ID保存在某一个文件中，如发送失败，等几分钟后重试  function sendPost() { var hasError = false; var tempArr = []; postListLength = voterIdList.length; console.log('send post');  if (voterIdList.length) { voterIdList.forEach(function (id, i) {  if (hasError) { // 处理发送失败的情况，等待5分钟重试 if (!sleepIng) { console.log('waiting'); sleepIng = true; return setTimeout(function () { sleepIng = false; sleep = 1; sendPost(); }, 1000 * 60 * 5); }  return console.log('has error'); }  var index = (function () { return i; })(i); var postIndex = index > postList.length ? index % postList.length : index;  setTimeout(function () { // 一波发送完成之前不会启动下一波私信发送 postListLength--; request.post('https://www.zhihu.com/inbox/post') .send({ member_id: id, content: postList[postIndex], token: '', _xsrf: '' // 这里是发送者的Cookie }) .set(config) .set({"Accept": "*/*"}) .set({"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"}) .end(function (err, res) { console.log('hasError: ', hasError); console.log(new Date()); console.log(res.text); var resObj = {};  try { resObj = JSON.parse(res.text); } catch (e) { console.log(e);  if (!sleepIng) { hasError = true; sleep = 5; console.log('waiting'); sleepIng = true; return setTimeout(function () { sleepIng = false; sleep = 1; sendPost(); }, 1000 * 60 * 5); } }  if (err || !res.ok || resObj.r !== 0) { console.log(err); hasError = true; sleep = 5; tempArr = voterIdList.slice(0, index); oldIdList = oldIdList.concat(tempArr); fs.writeFile('./idlist.json', oldIdList, function (err) { if (err) console.log(err); }); } }); }, 20 * 1000 * index * sleep);  if (index === voterIdList.length - 1) { console.log('last'); oldIdList = oldIdList.concat(voterIdList); voterIdList = []; setTimeout(function () { console.log('run again'); getVoterList(sourceLink, sendPost); }, 1000 * 60 * 15);  fs.writeFile('./idlist.json', oldIdList, function (err) { if (err) console.log(err); });  console.log('done '); } }); } else { setTimeout(function () { console.log('run again'); getVoterList(sourceLink, sendPost); }, 1000 * 60); } }      

       pip install Scrapy     

       pip install numpy     

       scrapy startproject acg     

       #coding=utf-8 #update at 2018-4-20 from acg.items import ImageItem import scrapy import numpy as np import os  class acgimages(scrapy.Spider):  """docstring for acgimages"""  name = 'images'  start_urls = [   "http://www.acg.fi/anime/page/1"  ]  page = 1  count = 0  MAX_CATCH_PAGES = 1000  item = ImageItem()  def parse(self,response):   next_page = response.xpath('//div[@class="grid-bor"]//a/@href').re(r'http://www.acg.fi/anime/([0-9]+).htm')   used = []   for page in next_page:    if page not in used:     used.append(page)   print('find %d secound pages' % len(used))   for number in used:    url = "http://www.acg.fi/anime/%s.htm" % number    self.item['url'] = url    yield scrapy.Request(url, callback = self.post_page)    if self.page < self.MAX_CATCH_PAGES:    self.page = self.page + 1   next_url = "http://www.acg.fi/anime/page/%d" % self.page   yield scrapy.Request(next_url, callback = self.parse)   def post_page(self,response):   images_url = response.xpath("//div[@id='entry-content']//img/@src").extract()   print('find %d images' % len(images_url))   self.item['images'] = images_url   return self.item     

       pip install pillow     

       import os from PIL import Image,ImageOps import argparse import time from multiprocessing import Pool import random import math import sys from colorsys import rgb_to_hsv  SLICE_SIZE = 85 OUT_SIZE = 5000 IN_DIR = "database/full/" OUT_DIR = "output/" REPATE = 0  def get_avg_color(img):     width, height = img.size     pixels = img.load()     if type(pixels) is not int:         data = []         for x in range(width):             for y in range(height):                 cpixel = pixels[x, y]                 data.append(cpixel)         h = 0         s = 0         v = 0         count = 0         for x in range(len(data)):             r = data[x][0]             g = data[x][1]             b = data[x][2]             count += 1             hsv = rgb_to_hsv(r / 255.0,g / 255.0,b / 255.0)             h += hsv[0]             s += hsv[1]             v += hsv[2]          hAvg = round(h / count,3)         sAvg = round(s / count,3)         vAvg = round(v / count,3)          if count > 0:              return (hAvg,sAvg,vAvg)         else:             raise IOError("读取图片数据失败")     else:         raise IOError("PIL 读取图片数据失败")   def find_closiest(color, list_colors):     diff = 1000     cur_closer = False     arr_len = 0     for cur_color in list_colors:         n_diff = math.sqrt(math.pow(math.fabs(color[0]-cur_color[0]), 2) + math.pow(math.fabs(color[1]-cur_color[1]), 2) + math.pow(math.fabs(color[2]-cur_color[2]), 2))         if n_diff < diff and cur_color[3] <= REPATE:             diff = n_diff             cur_closer = cur_color     if not cur_closer:         raise ValueError("没有足够的近似图片，建议设置重复")     cur_closer[3] += 1     return "({}, {}, {})".format(cur_closer[0],cur_closer[1],cur_closer[2])   def make_puzzle(img, color_list):     width, height = img.size     print("Width = {}, Height = {}".format(width,height))     background = Image.new('RGB', img.size, (255,255,255))     total_images = math.floor((width * height) / (SLICE_SIZE * SLICE_SIZE))     now_images = 0     for y1 in range(0, height, SLICE_SIZE):         for x1 in range(0, width, SLICE_SIZE):             try:                 y2 = y1 + SLICE_SIZE                 x2 = x1 + SLICE_SIZE                 new_img = img.crop((x1, y1, x2, y2))                 color = get_avg_color(new_img)                 close_img_name = find_closiest(color, color_list)                 close_img_name = OUT_DIR + str(close_img_name) + '.jpg'                 paste_img = Image.open(close_img_name)                 now_images += 1                 now_done = math.floor((now_images / total_images) * 100)                 r = '
[{}{}]{}%'.format("#"*now_done," " * (100 - now_done),now_done)                 sys.stdout.write(r)                                           sys.stdout.flush()                     background.paste(paste_img, (x1, y1))             except IOError:                 print('创建马赛克块失败')     return background   def get_image_paths():     paths = []     for file_ in os.listdir(IN_DIR):         paths.append(IN_DIR + file_)     if len(paths) > 0:         print("一共找到了%s" % len(paths) + "张图片")     else:         raise IOError("未找到任何图片")      return paths   def resize_pic(in_name,size):     img = Image.open(in_name)     img = ImageOps.fit(img, (size, size), Image.ANTIALIAS)     return img  def convert_image(path):     try:         img = resize_pic(path,SLICE_SIZE)         color = get_avg_color(img)         img.save(str(OUT_DIR) + str(color) + ".jpg")     except IOError:         print('图片处理失败')  def convert_all_images():     paths = get_image_paths()     print("正在生成马赛克块...")      pool = Pool()     pool.map(convert_image, paths)     pool.close()     pool.join()     def read_img_db():     img_db = []     for file_ in os.listdir(OUT_DIR):         if file_ == 'None.jpg':             pass         else:                  file_ = file_.split('.jpg')[0]             file_ = file_[1:-1].split(',')             file_ = list(map(float,file_))             file_.append(0)             print(file_)             img_db.append(file_)         return img_db  if __name__ == '__main__':      parse = argparse.ArgumentParser()     parse.add_argument("-i",'--input',required=True,help='input image')     parse.add_argument("-d", "--db", type=str, required=True,help="source database")     parse.add_argument("-o", "--output", type=str, required=True,help="out directory")     parse.add_argument("-s","--save",type=str,required=False,help="create image but not create database")     parse.add_argument("-is",'--inputSize',type=str, required=False,help="inputSize")     parse.add_argument("-os",'--outputSize',type=str, required=False,help="outputSize")     parse.add_argument("-r",'--repate',type=int, required=False,help="repate number")     args = parse.parse_args()     start_time = time.time()     args = parse.parse_args()     image = args.input      if args.db:         IN_DIR = args.db     if args.output:         OUT_DIR= args.output     if args.inputSize:         SLICE_SIZE = args.inputSize     if args.outputSize:         OUT_SIZE = args.outputSize     if not args.save:         convert_all_images()     if args.repate:         REPATE = args.repate      img = resize_pic(image,OUT_SIZE)     list_of_imgs = read_img_db()     out = make_puzzle(img, list_of_imgs)     img = Image.blend(out, img, 0.5)     img.save('out.jpg')      print("耗时: %s" % (time.time() - start_time))     print("已完成")       

       python puzzle.py -i test.jpg -d database/full/ -o output/