按用户爬取美拍视频

下载附件  保存到相册

2019-12-15 15:20 上传

function atob(input) {
    input = input.replace(/=+$/, '')
    if (input.length % 4 == 1) throw INVALID_CHARACTER_ERR;
    for (
        // initialize result and counters
        var bc = 0, bs, buffer, idx = 0, output = '';
        // get next character
        buffer = input.charAt(idx++);
        // character found in table? initialize bit storage and add its ascii value;
        ~buffer && (bs = bc % 4 ? bs * 64 + buffer : buffer,
            // and if not first of each 4 characters,
            // convert the first 8 bits to one ascii character
            bc++ % 4) ? output += String.fromCharCode(255 & bs >> (-2 * bc & 6)) : 0
    ) {
        // try to find character in table (0-63, not found => -1)
        var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
        buffer = chars.indexOf(buffer);
    }
    return output;
};

var decodeMp4 = {
    getHex: function (str) {
        return {
            str: str['substring'](4),
            hex: str['substring'](0, 4)['split']('').reverse().join('')
        };
    },
    getDec: function (hex) {
        var dec = parseInt(hex, 16).toString();
        return {
            pre: dec['substring'](0, 2)['split'](''),
            tail: dec['substring'](2)['split']('')
        };
    },
    substr: function (str, pos) {
        var str0 = str['substring'](0, pos[0]);
        var str1 = str['substr'](pos[0], pos[1]);
        return str0 + str['substring'](pos[0])['replace'](str1, '');
    },
    getPos: function (str, pos) {
        pos[0] = str.length - pos[0] - pos[1];
        return pos;
    },
    decode: function (str) {
        var result0 = this.getHex(str);
        var dec = this.getDec(result0.hex);
        var result1 = this['substr'](result0.str, dec.pre);
        return atob(this['substr'](result1, this.getPos(result1, dec.tail)));
    }
};
var origin_str = '0c91Ly9tdnMLmfNZpZGVvMTAubWVpdHVkYXRhLmNvbS81ZGJkNTE0OWU0YTc2N2dnMXNmbmltMTIwMl9IMjY0X01QNWRiZDUAKzLm1wNA==';
function decodeVideo(str){
    return decodeMp4.decode(str);
}
// console.log(decodeVideo(origin_str));
// console.log(decodeMp4.decode(origin_str));

import requests
from bs4 import BeautifulSoup
import re
import js2py
import time

# 用python运行js代码进行解密
def get_video_url(str):
    context = js2py.EvalJs()
    with open("meipai.js", "r", encoding="utf-8") as f:
        context.execute(f.read())
        result = context.decodeVideo(str)
        if not result[:4] =='http':
            result = 'http:'+ result
    return result

headers= {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'
}
def get_one_page(url):
    data_video_list = []
    response = requests.get(url,headers=headers)
    if response.status_code ==200:
        soup = BeautifulSoup(response.text,'lxml')
        mediasList = soup.select('#mediasList')[0]
        medias = mediasList.select('li.feed-item.pr')
        for media in medias:
            title = media.select('.detail-cover-title.break')
            description = media.select('.feed-description.break')
            if not description ==[]:
                description = description[0].get_text().strip()
                # print(description)
            if title ==[]:
                if description:
                    title = description
                else:
                    title = '无标题'
            else:
                title = title[0].string.strip()
            matched = re.search('data-video=\"(.*?)\">',str(media),re.S)
            if matched:
                # data-video="2301aHR0tcDovL212dmlkZW8xMS5tZWl0dWRhdGEuY29tLzVkZjBlMDYyNjI1NWNIMjY0V0VCMTgxMDA0X0gyNjRfTVA1ZGYwZjMuo3lsddbXA0">
                # return data-video
                video_info = {
                    'title':title,
                    'origin':matched.group(1)
                }
                # data_video_list.append(matched.group(1))
                data_video_list.append(video_info)

        return data_video_list

def get_total_page(url):
    response = requests.get(url,headers=headers)
    if response.status_code ==200:
        soup = BeautifulSoup(response.text,'lxml')                
        num_video = soup.select('#rightUser > div.user-num > a.user-num-item.user-hv.first.dbl > span.user-txt.pa')[0].string
        total_page = int(int(num_video) / 24)+1
        return total_page
        # print(total_page)

if __name__ == "__main__":
    # base_url = 'https://www.meipai.com/user/改成相应的id就可以采集其他用户的全部视频了'
    # base_url = 'https://www.meipai.com/user/35498115'
    base_url = 'https://www.meipai.com/user/30427636'
    total_page = get_total_page(base_url)
    print(total_page)
    for i in range(1,total_page+1):
        time.sleep(0.5)
        url = base_url + '?p={}'.format(str(i))
        data_video_list = get_one_page(url)
        for each_video in data_video_list:
            video_url = get_video_url(each_video['origin'])
            print(each_video['title'],video_url)