트위터에서 많은 팔로워를 크롤링하는 방법 [3]: 1초에 5천개 데이터 가져오기

import requests

cookies = {
    'dnt': '1',
    'kdt': 'testToken',
    'csrf_same_site_set': '1',
    'csrf_same_site': '1',
    'syndication_guest_id': 'testToken',
    '_ga': 'testToken',
    'personalization_id': 'testToken==',
    'guest_id': 'v1%testToken',
    'ads_prefs': 'testToken=',
    'remember_checked_on': '1',
    'u': 'testToken',
    'auth_token': 'testToken',
    'twid': 'u%testToken',
    'rweb_optin': 'side_no_out',
    'tfw_exp': '0',
    'des_opt_in': 'N',
    'night_mode': '0',
    'ct0': 'testToken',
}

headers = {
    'Connection': 'keep-alive',
    'Pragma': 'no-cache',
    'Cache-Control': 'no-cache',
    'Origin': 'https://twitter.com',
    'x-twitter-client-language': 'en',
    'x-csrf-token': 'testtoken',
    'authorization': 'Bearer testToken',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
    'x-twitter-auth-type': 'OAuth2Session',
    'x-twitter-active-user': 'yes',
    'DNT': '1',
    'Accept': '*/*',
    'Sec-Fetch-Site': 'same-site',
    'Sec-Fetch-Mode': 'cors',
    'Referer': 'https://twitter.com/twitter/followers',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
}

params = (
    ('include_profile_interstitial_type', '1'),
    ('include_blocking', '1'),
    ('include_blocked_by', '1'),
    ('include_followed_by', '1'),
    ('include_want_retweets', '1'),
    ('include_mute_edge', '1'),
    ('include_can_dm', '1'),
    ('include_can_media_tag', '1'),
    ('skip_status', '1'),
    ('cursor', '1653632830146283795'),
    ('user_id', '783214'),
    ('count', '20'),
)

response = requests.get('https://api.twitter.com/1.1/followers/list.json', headers=headers, params=params, cookies=cookies)

# 결과는 response.json()
result = response.json()
print(result)

('cursor', '1653632830146283795'),
('user_id', '783214'),
('count', '20'),

('cursor', '1653632830146283795'),
('screen_name', 'twitter'), # <-- 이 부분이 바뀌었다!
('count', '200'),

# ... 생략 ...

params = (
    ('include_profile_interstitial_type', '1'),
    ('include_blocking', '1'),
    ('include_blocked_by', '1'),
    ('include_followed_by', '1'),
    ('include_want_retweets', '1'),
    ('include_mute_edge', '1'),
    ('include_can_dm', '1'),
    ('include_can_media_tag', '1'),
    ('skip_status', '1'),
    ('cursor', cursor),
    ('user_id', user_id), # <-- screen_name으로 대체가능
    ('count', '5000'), # <-- max 5000
)

response = requests.get('https://api.twitter.com/1.1/folloewrs/ids.json', headers=headers, params=params)

import requests
import json
from bs4 import BeautifulSoup as bs

def get_follow_ids(user_id, username, cursor=-1):
    headers = {
        'authority': 'api.twitter.com',
        'origin': 'https://twitter.com',
        'x-twitter-client-language': 'ko',
        'x-csrf-token': '[TESTTOKEN]',
        'authorization': 'Bearer [TESTTOKEN]',
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
        'x-twitter-auth-type': 'OAuth2Session',
        'x-twitter-active-user': 'yes',
        'dnt': '1',
        'accept': '*/*',
        'sec-fetch-site': 'same-site',
        'sec-fetch-mode': 'cors',
        'referer': f'https://twitter.com/{username}/following',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
        'cookie': 'personalization_id="[TESTTOKEN]"; guest_id=v1%[TESTTOKEN]; ct0=[TESTTOKEN]; ads_prefs="[TESTTOKEN]="; kdt=[TESTTOKEN]; remember_checked_on=1; _twitter_sess=[TESTTOKEN]; auth_token=[TESTTOKEN]; csrf_same_site_set=1; rweb_optin=side_no_out; csrf_same_site=1; twid=u%[TESTTOKEN]',
    }

    params = (
        ('include_profile_interstitial_type', '1'),
        ('include_blocking', '1'),
        ('include_blocked_by', '1'),
        ('include_followed_by', '1'),
        ('include_want_retweets', '1'),
        ('include_mute_edge', '1'),
        ('include_can_dm', '1'),
        ('include_can_media_tag', '1'),
        ('skip_status', '1'),
        ('cursor', cursor),
        ('user_id', user_id),
        ('count', '5000'),
    )

    response = requests.get('https://api.twitter.com/1.1/followers/ids.json', headers=headers, params=params)
    return response.json()

def get_follow_all(user_id, username, user_follow_count=None):    
    cursor = -1
    dataset = []
    
    while cursor != 0: # 커서가 0이면 다음페이지가 없다.
        data = get_follow_ids(user_id, username, cursor)
        try:
            cursor = data['next_cursor']
        except KeyError as e:
            print(data)
            if data.get('error') and data['error'] == 'Not authorized.':
                open(f"protected/{user_id}", 'w').close()
                print(username, "Protected")
                return
            if data.get('errors') and data['errors'][0]['code'] == 34:
                open(f"deleted/{user_id}", 'w').close()
                print(username, "Does Not exist")
                return
            print('='*50)
            print('Username:', username, 'User_id', user_id)
            print(data)
            print('='*50)
            raise e
        dataset += data['ids']
        print(f'{username}({user_id}): {len(set(dataset))} / {user_follow_count}', end='\r')
    dataset = list(set(dataset))
    json.dump(dataset, open(f'success/{user_id}.json', 'w+'))
    print(f'# Final for USER [{username}({user_id})]: {len(dataset)} / {user_follow_count}')