Парсинг Reddit в 2023 году

subreddits = ['https://www.reddit.com/r/tech/top/?t=month']

class ScrapeReddit():
    def __init__(self):
        # start headless if you want later on.
        options = Options()
        self.driver = webdriver.Safari(service=Service(executable_path='/usr/bin/safaridriver'), options=options)
       
        self.postids = []
    
    def lazy_scroll(self):
        current_height = self.driver.execute_script('window.scrollTo(0,document.body.scrollHeight);')
        while True:
            self.driver.execute_script('window.scrollTo(0,document.body.scrollHeight);')
            time.sleep(2)
            new_height = self.driver.execute_script('return document.body.scrollHeight')   
            if new_height == current_height:      # this means we have reached the end of the page!
                html = self.driver.page_source
                break
            current_height = new_height
        return html

    def get_posts(self):
        for link in subreddits:
            self.driver.get(link)
            self.driver.maximize_window()
            time.sleep(5)
            html = self.lazy_scroll()
            # html = self.driver.page_source
            parser = BeautifulSoup(html, 'html.parser')
            post_links = parser.find_all('a', {'slot': 'full-post-link'})
            print(len(post_links))
            count = 1

            for post_link in post_links:
                # generate a unique id for each post
                post_id = post_link['href'].split('/')[-3]
                print(f"{count} - {post_id}")
                count += 1
                if post_id not in self.postids:
                    self.postids.append(post_id)

    def destroy(self):
        self.driver.close()

def get_data(self, postid):
    base_url = "https://reddit.com/"
    url = base_url + postid + ".json"
    self.driver.get(url)
    self.driver.maximize_window()
    html = self.driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    text = soup.find('body').get_text()
    time.sleep(3)
    return text
def get_post_details(self):
    jsons = []
    count = 1
    if not self.postids:
        print("No post ids found. Please run get_posts() first.")
        return
    for postid in self.postids:
        print(postid, count)
        text = self.get_data(postid)
        jsons.append(text)
        time.sleep(random.randint(1, 10))
        count += 1
    
    self.jsons = jsons
    return jsons

def get_post_info(json_data):
    """       Получает текст поста, все комментарии и их ответы,        идентификаторы пользователей поста, комментариев и ответов,        и временные метки поста, комментариев и ответов        из JSON данных.    """

    post = json_data[0]['data']['children'][0]['data']
    post_body = post['title']
    post_user = post['author']
    post_time = post['created_utc']
    comments = json_data[1]['data']['children']
    comments_list = []
    for (comment, idx) in zip(comments, range(len(comments))):
        comment_body = comment['data']['body']
        comment_user = comment['data']['author']
        comment_time = comment['data']['created_utc']
        comments_list.append({'body': comment_body,
                             'user': comment_user,
                             'time': comment_time})
        comment_replies = []

        # добавить ответ к комментарию, к которому он относится

        if comment['data']['replies'] != '':
            replies = comment['data']['replies']['data']['children']
            for reply in replies:
                reply_body = reply['data']['body']
                reply_user = reply['data']['author']
                reply_time = reply['data']['created_utc']
                comment_replies.append({'body': reply_body,
                        'user': reply_user, 'time': reply_time})
        comments_list[idx]['replies'] = comment_replies

    return {
        'post_body': post_body,
        'post_user': post_user,
        'post_time': post_time,
        'comments': comments_list,
        }

data = reddit.jsons

res = []
for i in range(len(data)):
  try:
    parsed_json = json.loads(data[i])
    res.append(get_post_info(parsed_json))
  except JSONDecodeError as e:
    print(e)
    continue

import os
from datetime import datetime

def save_to_json(data, subreddit):
    """save it as /data/{subreddit}/{timestamp}.json"""
    timestamp = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
    filename = f'data/{subreddit}/{timestamp}.json'
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, 'w') as f:
        json.dump(data, f)

save_to_json(res, reddit.subreddit)