Change to graphql api endpoint BirthdayCometRootQuery (#81)

mobeigi · web-flow · commit f773c5931b59 · 2020-11-12T19:21:26.000+11:00
Closes #24 (No locales being used anymore) Closes #32 (No async endpoint used so no domops) Closes #52 (No locales being used anymore) Closes #13 (No locales being used anymore)
diff --git a/README.md b/README.md
@@ -19,11 +19,9 @@ Around 20 June 2019, Facebook removed their Facebook Birthday ICS export option.
 This change was unannounced and no reason was ever released.  
 
 fb2cal is a tool which restores this functionality.  
-It works by calling various async endpoints that power the https://www.facebook.com/events/birthdays/ page.  
+It works by calling endpoints that power the https://www.facebook.com/events/birthdays/ page.  
 After gathering a list of birthdays for all the users friends for a full year, it creates a ICS calendar file. This ICS file can then be imported into third party tools (such as Google Calendar).
 
-This tool **does not** use the Facebook API.
-
 ## Requirements
 * Facebook account
 * Python 3.6+
@@ -59,9 +57,6 @@ It is recommended to run the script **once every 24 hours** to update the ICS fi
 ## Caveats
 * Facebook accounts secured with 2FA are currently not supported (see [#9](../../issues/9))
 * During Facebook authentication, a security checkpoint may trigger that will force you to change your Facebook password.
-* Some locales are currently not supported (see [#13](../../issues/13))
-* Some supported locales may fail. Consider changing your Facebook language to English temporarily as a workaround. (see [#52](../../issues/52))
-* Duplicate birthday events may appear if calendar is reimported after Facebook friends change their username due to performance optimizations. (see [#65](../../pull/65))
 
 ## Contributions
 Contributions are always welcome!
diff --git a/requirements.txt b/requirements.txt
@@ -1,8 +1,4 @@
 MechanicalSoup
 ics>=0.6
-babel
-pytz
 requests
-beautifulsoup4
-lxml
-python_dateutil
+lxml
diff --git a/src/birthday.py b/src/birthday.py
diff --git a/src/facebook_browser.py b/src/facebook_browser.py
@@ -2,19 +2,15 @@
 import re
 import requests
 import json
-from datetime import datetime
 from logger import Logger
-from utils import get_next_12_month_epoch_timestamps, strip_ajax_response_prefix
-import urllib.parse
-from transformer import Transformer
 
 class FacebookBrowser:
     def __init__(self):
         """ Initialize browser as needed """
         self.logger = Logger('fb2cal').getLogger()
         self.browser = mechanicalsoup.StatefulBrowser()
         self.browser.set_user_agent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36')
-        self.__cached_async_token = None
+        self.__cached_token = None
         self.__cached_locale = None
 
     def authenticate(self, email, password):
@@ -76,47 +72,15 @@ def authenticate(self, email, password):
             self.logger.error(f'Hit Facebook security checkpoint. Please login to Facebook manually and follow prompts to authorize this device.')
             raise SystemError
 
+    def get_token(self):
+        """ Get authorization token (CSRF protection token) that must be included in all requests """
 
-    def get_async_birthdays(self):
-        """ Returns list of birthday objects by querying the Facebook birthday async page """
-        
-        FACEBOOK_BIRTHDAY_ASYNC_ENDPOINT = 'https://www.facebook.com/async/birthdays/?'
-        birthdays = []
-        next_12_months_epoch_timestamps = get_next_12_month_epoch_timestamps()
-
-        transformer = Transformer()
-        user_locale = self.get_facebook_locale()
-
-        for epoch_timestamp in next_12_months_epoch_timestamps:
-            self.logger.info(f'Processing birthdays for month {datetime.fromtimestamp(epoch_timestamp).strftime("%B")}.')
-
-            # Not all fields are required for response to be given, required fields are date, fb_dtsg_ag and __a
-            query_params = {'date': epoch_timestamp,
-                            'fb_dtsg_ag': self.get_async_token(),
-                            '__a': '1'}
-            
-            response = self.browser.get(FACEBOOK_BIRTHDAY_ASYNC_ENDPOINT + urllib.parse.urlencode(query_params))
-            
-            if response.status_code != 200:
-                self.logger.debug(response.text)
-                self.logger.error(f'Failed to get async birthday response. Params: {query_params}. Status code: {response.status_code}.')
-                raise SystemError
-            
-            birthdays_for_month = transformer.parse_birthday_async_output(response.text, user_locale)
-            birthdays.extend(birthdays_for_month)
-            self.logger.info(f'Found {len(birthdays_for_month)} birthdays for month {datetime.fromtimestamp(epoch_timestamp).strftime("%B")}.')
-
-        return birthdays
-
-    def get_async_token(self):
-        """ Get async authorization token (CSRF protection token) that must be included in all async requests """
-
-        if self.__cached_async_token:
-            return self.__cached_async_token
-
-        FACEBOOK_BIRTHDAY_EVENT_PAGE_URL = 'https://www.facebook.com/events/birthdays/' # async token is present on this page
-        FACEBOOK_ASYNC_TOKEN_REGEXP_STRING = r'{\"token\":\".*?\",\"async_get_token\":\"(.*?)\"}'
-        regexp = re.compile(FACEBOOK_ASYNC_TOKEN_REGEXP_STRING, re.MULTILINE)
+        if self.__cached_token:
+            return self.__cached_token
+
+        FACEBOOK_BIRTHDAY_EVENT_PAGE_URL = 'https://www.facebook.com/events/birthdays/' # token is present on this page
+        FACEBOOK_TOKEN_REGEXP_STRING = r'{\"token\":\"(.*?)\"'
+        regexp = re.compile(FACEBOOK_TOKEN_REGEXP_STRING, re.MULTILINE)
 
         birthday_event_page = self.browser.get(FACEBOOK_BIRTHDAY_EVENT_PAGE_URL)
         
@@ -132,49 +96,36 @@ def get_async_token(self):
             self.logger.error(f'Match failed or unexpected number of regexp matches when trying to get async token.')
             raise SystemError
         
-        self.__cached_async_token = matches[1]
+        self.__cached_token = matches[1]
         
-        return self.__cached_async_token
+        return self.__cached_token
 
-    def get_facebook_locale(self):
-        """ Returns users Facebook locale """
-        
-        if self.__cached_locale:
-            return self.__cached_locale
+    def query_graph_ql_birthday_comet_root(self, offset_month):
+        """ Query the GraphQL BirthdayCometRootQuery endpoint that powers the https://www.facebook.com/events/birthdays page 
+            This endpoint will return all Birthdays for the offset_month plus the following 2 consecutive months. """
 
-        FACEBOOK_LOCALE_ENDPOINT = 'https://www.facebook.com/ajax/settings/language/account.php?'
-        FACEBOOK_LOCALE_REGEXP_STRING = r'[a-z]{2}_[A-Z]{2}'
-        regexp = re.compile(FACEBOOK_LOCALE_REGEXP_STRING, re.MULTILINE)
+        FACEBOOK_GRAPHQL_ENDPOINT = 'https://www.facebook.com/api/graphql/'
+        FACEBOOK_GRAPHQL_API_REQ_FRIENDLY_NAME = 'BirthdayCometRootQuery'
+        DOC_ID = 3382519521824494
 
-        # Not all fields are required for response to be given, required fields are fb_dtsg_ag and __a
-        query_params = {'fb_dtsg_ag': self.get_async_token(),
-                        '__a': '1'}
+        variables = {
+            'offset_month': offset_month,
+            'scale': 1.5
+        }
 
-        response = self.browser.get(FACEBOOK_LOCALE_ENDPOINT + urllib.parse.urlencode(query_params))
-        
-        if response.status_code != 200:
-            self.logger.debug(response.text)
-            self.logger.error(f'Failed to get Facebook locale. Params: {query_params}. Status code: {response.status_code}.')
-            raise SystemError
+        payload = {
+            'fb_api_req_friendly_name': FACEBOOK_GRAPHQL_API_REQ_FRIENDLY_NAME,
+            'variables': json.dumps(variables),
+            'doc_id': DOC_ID,
+            'fb_dtsg': self.get_token(),
+            '__a': '1'
+        }
 
-        # Parse json response
-        try:
-            json_response = json.loads(strip_ajax_response_prefix(response.text))
-            current_locale = json_response['jsmods']['require'][0][3][1]['currentLocale']
-        except json.decoder.JSONDecodeError as e:
-            self.logger.debug(response.text)
-            self.logger.error(f'JSONDecodeError: {e}')
-            raise SystemError
-        except KeyError as e:
-            self.logger.debug(json_response)
-            self.logger.error(f'KeyError: {e}')
-            raise SystemError
+        response = self.browser.post(FACEBOOK_GRAPHQL_ENDPOINT, data=payload)
 
-        # Validate locale
-        if not regexp.match(current_locale):
-            self.logger.error(f'Invalid Facebook locale fetched: {current_locale}.')
+        if response.status_code != 200:
+            self.logger.debug(response.text)
+            self.logger.error(f'Failed to get {FACEBOOK_GRAPHQL_API_REQ_FRIENDLY_NAME} response. Payload: {payload}. Status code: {response.status_code}.')
             raise SystemError
-
-        self.__cached_locale = current_locale
-
-        return self.__cached_locale
+        
+        return response.json()
diff --git a/src/facebook_user.py b/src/facebook_user.py
@@ -0,0 +1,14 @@
+class FacebookUser:
+    def __init__(self, id, name, profile_url, profile_picture_uri, birthday_day, birthday_month):
+        self.id = id
+        self.name = name
+        self.profile_url = profile_url
+        self.profile_picture_uri = profile_picture_uri
+        self.birthday_day = birthday_day
+        self.birthday_month = birthday_month
+
+    def __str__(self):
+        return f'{self.name} ({self.birthday_day}/{self.birthday_month})'
+    
+    def __unicode__(self):
+        return u'{self.name} ({self.birthday_day}/{self.birthday_month})'
diff --git a/src/fb2cal.py b/src/fb2cal.py
@@ -23,11 +23,11 @@
 import logging
 from distutils import util
 
-from birthday import Birthday
 from ics_writer import ICSWriter
 from logger import Logger
 from config import Config
 from facebook_browser import FacebookBrowser
+from transformer import Transformer
 
 if __name__ == '__main__':
     # Set CWD to script directory
@@ -62,18 +62,25 @@
         facebook_browser.authenticate(config['AUTH']['FB_EMAIL'], config['AUTH']['FB_PASS'])
         logger.info('Successfully authenticated with Facebook.')
 
-        # Get birthday objects for all friends via async endpoint
-        logger.info('Fetching all Birthdays via async endpoint...')
-        birthdays = facebook_browser.get_async_birthdays()
+        # Fetch birthdays for a full calendar year and transform them 
+        facebook_users = []
+        transformer = Transformer()
 
-        if len(birthdays) == 0:
-            logger.warning(f'Birthday list is empty. Failed to fetch any birthdays.')
+        # Endpoint will return all birthdays for offset_month plus the following 2 consecutive months.
+        logger.info('Fetching all Birthdays via BirthdayCometRootQuery endpoint...')
+        for offset_month in [1, 4, 7, 10]:
+            birthday_comet_root_json = facebook_browser.query_graph_ql_birthday_comet_root(offset_month)
+            facebook_users_for_quarter = transformer.transform_birthday_comet_root_to_birthdays(birthday_comet_root_json)
+            facebook_users.extend(facebook_users_for_quarter)
+
+        if len(facebook_users) == 0:
+            logger.warning(f'Facebook user list is empty. Failed to fetch any birthdays.')
             raise SystemError
 
-        logger.info(f'A total of {len(birthdays)} birthdays were found.')
+        logger.info(f'A total of {len(facebook_users)} birthdays were found.')
 
         # Generate ICS
-        ics_writer = ICSWriter(birthdays)
+        ics_writer = ICSWriter(facebook_users)
         logger.info('Creating birthday ICS file...')
         ics_writer.generate()
         logger.info('ICS file created successfully.')
diff --git a/src/ics_writer.py b/src/ics_writer.py
@@ -12,9 +12,9 @@
 """ Write Birthdays to an ICS file """
 class ICSWriter:
 
-    def __init__(self, birthdays):
+    def __init__(self, facebook_users):
         self.logger = Logger('fb2cal').getLogger()
-        self.birthdays = birthdays
+        self.facebook_users = facebook_users
 
     def generate(self):
         c = Calendar()
@@ -27,23 +27,23 @@ def generate(self):
 
         cur_date = datetime.now()
 
-        for birthday in self.birthdays:
+        for facebook_user in self.facebook_users:
             e = Event()
-            e.uid = birthday.uid
+            e.uid = facebook_user.id
             e.created = cur_date
-            e.name = f"{birthday.name}'s Birthday"
+            e.name = f"{facebook_user.name}'s Birthday"
 
             # Calculate the year as this year or next year based on if its past current month or not
             # Also pad day, month with leading zeros to 2dp
-            year = cur_date.year if birthday.month >= cur_date.month else (cur_date + relativedelta(years=1)).year
+            year = cur_date.year if facebook_user.birthday_month >= cur_date.month else (cur_date + relativedelta(years=1)).year
             
             # Feb 29 special case: 
             # If event year is not a leap year, use Feb 28 as birthday date instead
-            if birthday.month == 2 and birthday.day == 29 and not calendar.isleap(year):
-                birthday.day = 28
+            if facebook_user.birthday_month == 2 and facebook_user.birthday_day == 29 and not calendar.isleap(year):
+                facebook_user.birthday_day = 28
 
-            month = '{:02d}'.format(birthday.month)
-            day = '{:02d}'.format(birthday.day)
+            month = '{:02d}'.format(facebook_user.birthday_month)
+            day = '{:02d}'.format(facebook_user.birthday_day)
             e.begin = f'{year}-{month}-{day} 00:00:00'
             e.make_all_day()
             e.duration = timedelta(days=1)
diff --git a/src/transformer.py b/src/transformer.py
diff --git a/src/utils.py b/src/utils.py
