SCRIPT_SAVE_PATH = os.path.join(os.getcwd(), 'script_save.pkl')
script_save = None
try:
with open(SCRIPT_SAVE_PATH, 'rb') as f:
script_save = pickle.load(f)
print("Loaded script save. Resuming...")
print("NUM_POSTS_FROM_SUB:", script_save["NUM_POSTS_FROM_SUB"])
print("NUM_POSTS_OF_USER:", script_save["NUM_POSTS_OF_USER"])
print("MIN_TIMES_POSTED:", script_save["MIN_TIMES_POSTED"])
print("MAX_DEPTH:", script_save["MAX_DEPTH"])
print("Number of subreddits in queue:", len(script_save["sub_q"]))
print("Number of posts saved so far:", script_save["num_posts_saved"])
except:
print("No script save found. Starting from scratch...")
NUM_POSTS_FROM_SUB = 500 if script_save is None else script_save["NUM_POSTS_FROM_SUB"]
NUM_POSTS_OF_USER = 5 if script_save is None else script_save["NUM_POSTS_OF_USER"]
MIN_TIMES_POSTED = 2 if script_save is None else script_save["MIN_TIMES_POSTED"]
MAX_DEPTH = 5 if script_save is None else script_save["MAX_DEPTH"]
sub_q = ["programming"] if script_save is None else script_save["sub_q"]
sub_depths = {sub_q[0]: 0} if script_save is None else script_save["sub_depths"]
skipped_subs = [] if script_save is None else script_save["skipped_subs"]
reddit = praw.Reddit(client_id=CLIENT_ID, client_secret=CLIENT_SECRET, user_agent=USER_AGENT)
num_posts_saved = 0 if script_save is None else script_save["num_posts_saved"]
# BFS; take the first subreddit from the queue.
while len(sub_q) > 0 and (sub:=sub_q.pop(0)):
print("=========================================")
print(f"Processing '{sub}' on depth {sub_depths[sub]}")
print(f"Queue size: {len(sub_q)}")
print(f"Num posts saved so far: {num_posts_saved}")
posts = None
try:
# Download posts from the subreddit
posts = list(reddit.subreddit(sub).top(limit=NUM_POSTS_FROM_SUB, time_filter="all"))
except:
print(f"ERROR: Cannot access '{sub}'")
skipped_subs.append(sub)
if posts is not None:
if len(posts) < NUM_POSTS_FROM_SUB:
print(f"Only {len(posts)} posts found")
data_df = pd.DataFrame(
[[post.title, post.score, post.id, post.url, post.num_comments, post.created, post.author, post.upvote_ratio, post.permalink, post.subreddit, post.subreddit_subscribers, sub_depths[sub]] for post in posts],
columns=["title", "score", "id", "url", "num_comments", "created", "author", "upvote_ratio", "permalink", "subreddit", "subreddit_subscribers", "depth"],
)
# Filter out posts made by deleted users
data_df = data_df[data_df["author"].notna()]
# Keep only authors that posted more then MIN_TIMES_POSTED times
data_df["author_name"] = data_df["author"].apply(lambda x: x.name)
data_df = data_df.groupby("author_name").filter(lambda x: len(x) >= MIN_TIMES_POSTED)
data_df = data_df.drop(columns=["author_name"])
authors = data_df["author"].unique()
num_posts = len(data_df)
num_posts_saved += num_posts
print(f"Num posts after filtering out: {num_posts} from {len(authors)} authors")
# Check if we reached the max depth
if sub_depths[sub] >= MAX_DEPTH:
print("Max depth reached")
else:
for author in authors:
try:
# Try to get submissions of the author
user_submissions = list(author.submissions.top(limit=NUM_POSTS_OF_USER, time_filter="all"))
# Extract subreddits from top user submissions and add them to the queue
for submission in user_submissions:
sub_name = submission.subreddit.display_name
if sub_name not in sub_depths:
sub_q.append(sub_name)
sub_depths[sub_name] = sub_depths[sub] + 1
except:
print(f"ERROR: User submissions are private for '{author}'")
# Save dataframe to csv
data_df.to_csv(f"{DATA_PATH}/posts_{sub}.csv", index=False)
# Save the script state to be able to resume in case of an error
script_save = {
"NUM_POSTS_FROM_SUB": NUM_POSTS_FROM_SUB,
"NUM_POSTS_OF_USER": NUM_POSTS_OF_USER,
"MIN_TIMES_POSTED": MIN_TIMES_POSTED,
"MAX_DEPTH": MAX_DEPTH,
"sub_q": sub_q,
"sub_depths": sub_depths,
"num_posts_saved": num_posts_saved,
"skipped_subs": skipped_subs,
}
with open(SCRIPT_SAVE_PATH, 'wb') as f:
pickle.dump(script_save, f)
