vergnet/application-amicale-serveur
1
0
Fork 1
Code Issues Pull requests 1 Releases Wiki Activity

Add support for multiple page scraping

Browse source
This commit is contained in:
Arnaud Vergnet 2020-08-08 15:53:54 +02:00
parent 77c4bb01d8
commit f0de22a8b1
1 changed files with 12 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
facebook/facebook_handler.py
View file

@ -1,12 +1,14 @@
import json import json
import facebook_scraper import facebook_scraper
import enum
FILE = 'facebook_data.json' FILE = 'facebook_data.json'
PAGES = ["amicale.deseleves", "campus.insat"]
def scrape_data(): def scrape_data(page):
post_list = [] post_list = []
for post in facebook_scraper.get_posts('amicale.deseleves', pages=3): for post in facebook_scraper.get_posts(page, pages=3):
print(post) print(post)
cleaned_post = { cleaned_post = {
"id": post["post_id"], "id": post["post_id"],
@ -21,6 +23,13 @@ def scrape_data():
return post_list return post_list
def get_all_data():
data = {}
for page in PAGES:
data[page] = scrape_data(page)
return data
def write_data(data): def write_data(data):
with open(FILE, 'w') as f: with open(FILE, 'w') as f:
json.dump(data, f) json.dump(data, f)
@ -28,7 +37,7 @@ def write_data(data):
def main(): def main():
print("Fetching facebook data...") print("Fetching facebook data...")
write_data(scrape_data()) write_data(get_all_data())
print('DONE') print('DONE')