Merge pull request #7 from adizafri2000/dev

PR Dev to main
adizafri2000 · Sep 4, 2023 · 582e7d1 · 582e7d1
2 parents 995b19e + 77b8537
commit 582e7d1
Show file tree

Hide file tree

Showing 9 changed files with 193 additions and 65 deletions.
diff --git a/.github/workflows/build-dev-branch.yml b/.github/workflows/build-dev-branch.yml
@@ -0,0 +1,48 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Dev branch build & test
+
+on:
+  push:
+    branches: [ "dev" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-test:
+    env:
+      DISPLAY: :0
+      TNB_EMAIL: ${{ secrets.TNB_EMAIL }}
+      TNB_PASSWORD: ${{ secrets.TNB_PASSWORD }}
+      AIR_EMAIL: ${{ secrets.AIR_EMAIL }}
+      AIR_PASSWORD: ${{ secrets.AIR_PASSWORD }}
+      mynum: ${{ secrets.mynum }}
+      ws_group_id: ${{ secrets.ws_group_id }}
+
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Allow permissions to execute shell scripts
+        run: sudo chmod +x setup.sh runner.sh
+        shell: sh
+
+      - name: Setup headless display
+        uses: pyvista/setup-headless-display-action@v1
+
+      - name: Set up terminal dependencies.
+        run: ./setup.sh
+        shell: sh
+
+      - name: xhost setup
+        run: xhost +si:localuser:runner
+        shell: sh
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+
diff --git a/automation.py b/automation.py
@@ -3,6 +3,7 @@
 
 from dotenv import load_dotenv
 from selenium import webdriver
+from selenium.common import NoSuchElementException
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
@@ -17,6 +18,9 @@
 AIR_DASHBOARD_URL = 'https://crisportal.airselangor.com/profile/dashboard?lang=en'
 SCREENSHOT_DIR = "screenshots"
 
+BILL_TNB = "tnb"
+BILL_AIR = "air"
+
 # TODO Add logouts to all the automation logic
 # TODO Clean up the mess(es) (basically everything)
 
@@ -30,16 +34,36 @@ def generate_scshot_name(bill_type):
     return SCREENSHOT_DIR + os.sep + folder + os.sep + f"{folder}-" + datetime.now().strftime("%Y%m%d-%H%M%S") + ".png"
 
 
+def generate_screenshot(driver, bill_type):
+    img_name = generate_scshot_name(bill_type)
+    logger.info(f"Saving image to {img_name}")
+    driver.save_screenshot(img_name)
+
 # TODO Add a checkpoint in the logic to return e.g. False to caller
 #  in case of failed automation
 def automate_tnb(driver: webdriver.Chrome) -> {}:
+
+    wait = WebDriverWait(driver, 20)
     driver.get(TNB_URL)
     logger.info(f"Current browser URL: {driver.current_url}")
 
-    tnb_email_input = driver.find_element(By.NAME, "email")
-    tnb_password_input = driver.find_element(By.NAME, "password")
-    tnb_login_button = driver.find_element(By.XPATH,
-                                           "//*[@id=\"frm-login\"]/div[2]/div/div[2]/div/div[5]/div[2]/button")
+    try:
+        # tnb_email_input = driver.find_element(By.NAME, "email")
+        # print(f'tnb_email_input displayed: {tnb_email_input.is_displayed()}')
+        merdeka_popup = driver.find_element(By.XPATH, "/html/body/div[2]/div/div/div/div[2]/button")
+        print(f'merdeka_popup displayed: {merdeka_popup.is_displayed()}')
+        merdeka_popup.click()
+    except:
+        #merdeka popup
+        # merdeka_popup = driver.find_element(By.XPATH, "/html/body/div[2]/div/div/div/div[2]/button")
+        # print(f'merdeka_popup displayed: {merdeka_popup.is_displayed()}')
+        # merdeka_popup.click()
+        pass
+    finally:
+        tnb_email_input = driver.find_element(By.NAME, "email")
+        tnb_password_input = driver.find_element(By.NAME, "password")
+        tnb_login_button = driver.find_element(By.XPATH,
+                                               "//*[@id=\"frm-login\"]/div[2]/div/div[2]/div/div[5]/div[2]/button")
 
     tnb_email = os.getenv("TNB_EMAIL")
     tnb_password = os.getenv("TNB_PASSWORD")
@@ -63,14 +87,15 @@ def automate_tnb(driver: webdriver.Chrome) -> {}:
         "to_pay" : "//*[@id=\"mainBody\"]/div[5]/div[1]/div/div/div/div[2]/div[2]/div[2]/div/span[2]",
         "latest_bill" : "//*[@id=\"mainBody\"]/div[5]/div[1]/div/div/div/div[2]/div[1]/div[3]/div[2]/label",
         "outstanding_charges" : "//*[@id=\"mainBody\"]/div[5]/div[1]/div/div/div/div[2]/div[1]/div[4]/div[2]/label",
-        "popup_later_button_xpath" : "//*[@id=\"modal-button-1\"]/div/button[1]"
+        "popup_later_button_xpath" : "//*[@id=\"modal-button-2\"]/div/button[1]", #if doesnt work, change to 1,
+        "last_payment" : "//*[@id=\"mainBody\"]/div[5]/div[2]/div/div/div/div[2]/div/div[2]"
     }
 
     logger.info(f"Current browser URL: {driver.current_url}")
 
     try:
         logger.info("Attempting to find popup...")
-        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, xpath.get("popup_later_button_xpath"))))
+        wait.until(EC.presence_of_element_located((By.XPATH, xpath.get("popup_later_button_xpath"))))
         popup_later_button = driver.find_element(By.XPATH, xpath.get("popup_later_button_xpath"))
         popup_later_button.click()
         logger.info("Popup found & closed")
@@ -80,17 +105,20 @@ def automate_tnb(driver: webdriver.Chrome) -> {}:
             (By.XPATH, xpath.get("to_pay"))))
     finally:
         driver.implicitly_wait(7)
+        WebDriverWait(driver,20)
+
+        generate_screenshot(driver, BILL_TNB)
 
         to_pay = driver.find_element(By.XPATH, xpath.get("to_pay")).text
         bill_date = driver.find_element(By.XPATH, xpath.get("bill_date")).text
 
         # different presented UI and needed steps if bill already paid
-        if to_pay=="0.00":
-            latest_bill = driver.find_element(By.XPATH,xpath.get("latest_bill")).text.split()[1]
+        if to_pay == "0.00":
+            latest_bill = driver.find_element(By.XPATH,xpath.get("last_payment")).text.split()[1]
             outstanding_charges = driver.find_element(By.XPATH, xpath.get("outstanding_charges")).text.split()[1]
             msg = (f"TNB Bill Scraped Data:\n"
             f"Bill Date: {bill_date}\n"
-            f"Outstanding Charges: RM{outstanding_charges}\n"
+            # f"Outstanding Charges: RM{outstanding_charges}\n"
             f"Latest Bill: RM{latest_bill}\n"
             f"Bill has been paid!")
 
@@ -100,7 +128,11 @@ def automate_tnb(driver: webdriver.Chrome) -> {}:
         else:
             prev_balance = driver.find_element(By.XPATH, xpath.get("prev_balance")).text.split()[1]
             current_charges = driver.find_element(By.XPATH, xpath.get("current_charges")).text.split()[1]
-            rounding_adj = driver.find_element(By.XPATH, xpath.get("rounding_adj")).text.split()[1]
+            try:
+                rounding_adj = driver.find_element(By.XPATH, xpath.get("rounding_adj")).text.split()[1]
+            except NoSuchElementException:
+                logger.info("No rounding adjustment found (no theft), setting value to 0")
+                rounding_adj = 0.00
             msg = (f"TNB Bill Scraped Data:\n"
             f"Bill Date: {bill_date}\n"
             f"Previous Balance: RM{prev_balance}\n"
@@ -110,19 +142,6 @@ def automate_tnb(driver: webdriver.Chrome) -> {}:
 
         logger.info(msg)
 
-    img_name = generate_scshot_name("tnb")
-    logger.info(f"Saving image to {img_name}")
-    driver.save_screenshot(img_name)
-
-    # logging out
-    '''
-    path_logout_button = "//*[@id=\"logout\"]/p/span"
-    driver.find_element(By.XPATH,path_logout_button).click()
-
-    if driver.current_url=="https://www.mytnb.com.my/":
-        logger.info("Successfully logged out")
-    '''
-
     return {
         "type": "Elektrik",
         "to_pay": to_pay,
@@ -140,10 +159,8 @@ def automate_air(driver: webdriver.Chrome) -> {}:
         popup_close_button = driver.find_element(By.XPATH, "//*[@id=\"__layout\"]/div/div[2]/div/div/span/i")
         popup_close_button.click()
         logger.info("Popup found and closed!")
-        driver.save_screenshot(generate_scshot_name("air"))
     except:
         logger.info("No popup found!")
-        driver.save_screenshot(generate_scshot_name("air"))
         pass
 
     driver.implicitly_wait(5)
@@ -167,15 +184,13 @@ def automate_air(driver: webdriver.Chrome) -> {}:
 
     logger.info(f"Current browser URL: {driver.current_url}")
 
+    generate_screenshot(driver, BILL_AIR)
+
     to_pay = driver.find_element(By.XPATH, "//*[@id=\"printBill\"]/tbody/tr[1]/td[5]").text
     bill_date = driver.find_element(By.XPATH, "//*[@id=\"printBill\"]/tbody/tr[1]/td[3]").text
 
     logger.info(f"Air Selangor Bill Scraped Data:\nBill Date: {bill_date}\nTo pay: RM{to_pay}")
 
-    img_name = generate_scshot_name("air")
-    logger.info(f"Saving image to {img_name}")
-    driver.save_screenshot(img_name)
-
     return {
         "type": "Air",
         "to_pay": to_pay,
@@ -194,3 +209,4 @@ def generate_internet_bill():
         "bill_date": datetoday,
         "retrieved_date": datetime.now().strftime("%Y%m%d-%H%M%S")
     }
+
diff --git a/main.py b/main.py
@@ -11,6 +11,9 @@
 import whatsapp
 from main_logging import logger
 
+# to avoid KeyError 'Display' when importing pywhatkit
+# os.environ['DISPLAY'] = ':0'
+
 load_dotenv()
 data = []
 
@@ -21,7 +24,7 @@ def clean_resources(driver: webdriver.Chrome):
     logger.info("Closed and quit webdriver connection!")
 
 
-def get_chromedriver():
+def get_chromedriver(headless=False):
     """Returns a chromedriver executable based on detected machine OS"""
     driver_dir = "chromedriver" + os.sep
     if platform.system().lower() != "windows":
@@ -30,7 +33,19 @@ def get_chromedriver():
     else:
         driver_dir += "WIN_32_chromedriver.exe"
     logger.info("Using chromedriver from {}".format(driver_dir))
-    return webdriver.Chrome(driver_dir)
+
+    option = webdriver.ChromeOptions()
+    if headless:
+        logger.info("Running chromedriver in headless mode")
+        option.add_argument("--headless=new")
+    else:
+        logger.info("Running chromedriver in normal GUI mode")
+
+    option.add_argument('--disable-gpu')
+    option.add_argument('--no-sandbox')
+    option.add_argument("--window-size=1920x1080")
+
+    return webdriver.Chrome(driver_dir, options=option)
 
 
 def get_chromedriver_by_service(headless=False):
@@ -46,7 +61,8 @@ def get_chromedriver_by_service(headless=False):
     option.add_argument("--window-size=1920x1080")
 
     return webdriver.Chrome(
-        service=Service(ChromeDriverManager().install()),
+        #service=Service(ChromeDriverManager().install()),
+        service=Service(),
         options=option
     )
 
@@ -70,7 +86,7 @@ def main():
 
     msg = whatsapp.generate_message(data)
     logger.info(msg)
-    #whatsapp.send_whatsapp_to_me(msg)
+    whatsapp.send_whatsapp_to_me(msg)
     #whatsapp.send_whatsapp_group(msg)
 
     clean_resources(driver)

diff --git a/readme.md b/readme.md
@@ -3,6 +3,8 @@
 Headless (partly) automation script to scrape bills, calculate monthly bill (rent + utilities) and generate a text message which is later
 sent to desired whatsapp recipients.
 
+[![Dev branch build & test](https://github.com/adizafri2000/billscraper/actions/workflows/build-dev-branch.yml/badge.svg)](https://github.com/adizafri2000/billscraper/actions/workflows/build-dev-branch.yml)
+
 ## Features
 1. Scrapes bills from [TNB portal](https://www.mytnb.com.my/) and [Air Selangor portal](https://crisportal.airselangor.com/?lang=en).
 2. Calculates monthly bill (rent + utilities) and splits the total price among the total number of housemates.
@@ -11,19 +13,17 @@ sent to desired whatsapp recipients.
 ## Requirements and supported platforms
 1. Python (developed on v3.9)
 2. Ubuntu 20.04 (developed on Windows machine with Ubuntu WSL)
-3. A [MongoDB Atlas](https://www.mongodb.com/cloud/atlas/register) account and database instance
-4. A working phone number with a Whatsapp account
+3A working phone number with a Whatsapp account
 
 It is not guaranteed that the automation can run on different platforms, but it isn't guaranteed that it would not
 be able to run either. If you are using an older/later python version or different Linux distro/version but it still
 runs, you're good to go!
 
 ## Limitations
-1. Yet to test if the headless part of the automation is truly headless.
-2. The whatsapp message sending module of the app will still open a Chrome window to send the message, hence **dependent on a display**.
+1. The whatsapp message sending module of the app will still open a Chrome window to send the message, hence **dependent on a display** and making the entire thing *not headless*.
 3. Pywhatkit library **may not successfully send the message** sometimes. Sometimes, it opens [Whatsapp Web](https://web.whatsapp.com/), writes the message, but doesn't send it.
-4. If the **HTML structure of [TNB portal](https://www.mytnb.com.my/) or [Air Selangor portal](https://crisportal.airselangor.com/?lang=en) changes** or either **websites are down**, the automation will fail.
-5. If **Pywhatkit fails**, the automation fails too.
+4. If the **HTML structure of [TNB portal](https://www.mytnb.com.my/) or [Air Selangor portal](https://crisportal.airselangor.com/?lang=en) changes** or either **websites are down**, the automation will fail. (**Seasonal web popups** that are not expected by the automation can cause it to fail too)
+5. If **Pywhatkit fails**, the automation fails too. This is basically the case for all the core dependencies of the automation.
 
 ## How it Works
 1. A Selenium webdriver for chrome is initialized, and will 'drive' on Google Chrome.
@@ -59,10 +59,6 @@ AIR_EMAIL=[your Air Selangor account email]
 AIR_PASSWORD=[your Air Selangor account password]
 mynum=[your Whatsapp phone number]
 ws_group_id=[your whatsapp group id]
-
-DB_URL=[your MongoDB Atlas database instance/cluster url]
-DB_USERNAME=[your MongoDB Atlas database instance/cluster username]
-DB_PASSWORD=[your MongoDB Atlas database instance/cluster password]
 ````
 
 Replace the placeholder details with your details. Make sure the remove the squared brackets, and leave no spaces

diff --git a/requirements.txt b/requirements.txt
@@ -2,4 +2,5 @@ selenium
 python-dotenv
 pywhatkit
 webdriver_manager
-pyautogui
+pyautogui
+twilio
diff --git a/runner.sh b/runner.sh
@@ -2,16 +2,25 @@
 
 #xvfb-run java -Dwebdriver.chrome.driver=chromedriver/chromedriver_linux64_114-0-5735-16 -jar chromedriver/selenium-server-standalone.jar
 
-echo "(1/5) Activating venv"
-. venv/bin/activate
+echo "(1/6) Activating venv"
+. venv/Scripts/activate
 
-echo "(2/5) Executing automation script"
+echo "(2/6) Installing dependencies from requirements.txt"
+pip install -r requirements.txt
+
+echo "(3/6) Executing automation script"
 python3 main.py
 
-echo "(3/5) Deactivating venv"
+#if $?!=0
+#then
+#  echo "Program exited with non-zero status code and failed."
+#  set -e
+#fi
+
+echo "(4/6) Deactivating venv"
 deactivate
 
-echo "(4/5) Cleaning up processes"
+echo "(5/6) Cleaning up processes"
 kill $(ps ax | grep dbus-launch | awk '{print $1}' | head -n 1)
 
-echo "(5/5) Finished! (Should be)"
+echo "(6/6) Finished! (Should be)"