終於來到這次學習 Telegram API 的倒數第二站 : 修復我的市圖爬蟲, 自從 LINE Notify 終止服務後就沒辦法掌握借書還書與預約狀況, 必須花時間手動上網 (其實早在年初就發現沒收到訊息~網頁些微改版). 此爬蟲使用 Selenium 來模擬真人操作網頁, 參考 :
關於 Telgram Bot API 用法參考下面的索引 :
本篇旨在將原先市圖借書與預約爬蟲推播到 LINE 改成推播到 Telegram, 同時因應圖書館網頁改版修改了部分程式碼, 其實只是改登入頁面中帳號欄位的 id 而已 :
新版網頁把登入帳號框的 id 由 loginid 改為 logxinid, 其餘都沒變 (可能就是偶而要搞一下爬蟲設計者哈哈). 原程式 ksml_personal_8.py 修改為如下之 ksml_personal_9.py :
# ksml_personal_9.py
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
import re
from datetime import datetime
import time
import requests
import asyncio
from telegram import Bot
async def telegram_send_text(text):
bot=Bot(token=token)
try:
await bot.send_message(
chat_id=chat_id,
text=text
)
return True
except Exception as e:
print(f'Error sending text: {e}')
return False
def get_books(account, password):
try:
# 登入我的書房
options=Options()
options.add_argument("--headless")
driverpath='geckodriver.exe'
browser=webdriver.Firefox(options=options)
browser.implicitly_wait(60)
browser.get('https://webpacx.ksml.edu.tw/personal/')
loginid=browser.find_element(By.ID, 'logxinid')
loginid.send_keys(account)
pincode=browser.find_element(By.ID, 'pincode')
pincode.send_keys(password)
div_btn_grp=browser.find_element(By.CLASS_NAME, 'btn_grp')
login_btn=div_btn_grp.find_element(By.TAG_NAME, 'input')
login_btn.click()
# 擷取借閱紀錄
div_redblock=browser.find_element(By.CLASS_NAME, 'redblock')
div_redblock.click()
books=browser.find_elements(By.CLASS_NAME, 'bookdata')
borrow_books=[]
for book in books:
item=dict()
book_name=book.find_element(By.XPATH, './h2/a').text
item['book_name']=book_name.replace('/', '').strip()
reg=r'\d{4}-\d{2}-\d{2}'
due_date=book.find_element(By.XPATH, './ul[4]/li[2]').text
item['due_date']=re.findall(reg, due_date)[0]
due_times=book.find_element(By.XPATH, './ul[5]/li[1]').text
item['due_times']=re.findall(r'\d{1}', due_times)[0]
try:
state=book.find_element(By.XPATH, './ul[6]/li[1]').text
except:
state=''
finally:
if '有人預約' in state:
item['state']=', 有人預約'
else:
item['state']=''
borrow_books.append(item)
browser.back() # 回上一頁
# 擷取預約紀錄
div_blueblock=browser.find_element(By.CLASS_NAME, 'blueblock')
div_blueblock.click()
books=browser.find_elements(By.CLASS_NAME, 'bookdata')
reserve_books=[]
for book in books:
item=dict()
book_name=book.find_element(By.XPATH, './h2/a').text
item['book_name']=book_name.replace('/', '').strip()
sequence=book.find_element(By.XPATH, './ul[7]/li[1]').text
if '預約待取' in sequence: # 已到館
item['ready_for_pickup']=True
reg=r'\d{4}-\d{2}-\d{2}'
item['expiration']=re.findall(reg, sequence)[0]
item['sequence']='0'
else: # 預約中
item['ready_for_pickup']=False
item['expiration']=''
item['sequence']=re.findall(r'\d+', sequence)[0]
reserve_books.append(item)
browser.close()
return (borrow_books, reserve_books)
except Exception as e:
print(e)
return None, None
if __name__ == '__main__':
start=time.time()
token='我的權杖'
chat_id='我的聊天室 ID'
users=[['帳號1', '密碼1'],
['帳號2', '密碼2']]
for user in users:
account=user[0]
password=user[1]
borrow_books, reserve_books=get_books(account, password)
if borrow_books:
borrow=[]
for book in borrow_books:
book_name=book['book_name']
due_times=book['due_times']
due_date=book['due_date']
state=book['state']
due_date=datetime.strptime(due_date, '%Y-%m-%d')
today_str=datetime.today().strftime('%Y-%m-%d')
today=datetime.strptime(today_str, "%Y-%m-%d")
delta=(due_date-today).days
if delta < 0:
msg=f'🅧 {book_name} (逾期 {abs(delta)} 天{state})'
borrow.append(msg)
elif delta == 0:
msg=f'⓿ {book_name} (今日到期, 續借次數 {due_times}{state})'
borrow.append(msg)
elif delta == 1:
msg=f'❶ {book_name} (明日到期, 續借次數 {due_times}{state})'
borrow.append(msg)
elif delta == 2:
msg=f'❷ {book_name} (後天到期, 續借次數 {due_times}{state})'
borrow.append(msg)
elif 2 < delta < 8:
msg=f'✦ {book_name} ({book["due_date"]} 到期, '\
f'續借次數 {due_times}{state})'
borrow.append(msg)
if len(borrow) != 0:
borrow.insert(0, f'\n❖ {account} 的借閱 :')
msg='\n'.join(borrow)
if asyncio.run(telegram_send_text(msg)):
print('訊息傳送成功!')
else:
print('訊息傳送失敗!')
if reserve_books:
reserve=[]
i=0
j=['①', '②', '③', '④', '⑤']
k=['❶', '❷', '❸', '❹', '❺']
for book in reserve_books:
book_name=book['book_name']
sequence=book['sequence']
ready_for_pickup=book['ready_for_pickup']
expiration=book['expiration']
if ready_for_pickup:
msg=f'{k[i]} {book_name} (已到館, 保留期限 {expiration})'
else:
msg=f'{j[i]} {book_name} (順位 {sequence})'
reserve.append(msg)
i += 1
if len(reserve) != 0:
reserve.insert(0, f'\n❖ {account} 的預約 :')
msg='\n'.join(reserve)
if asyncio.run(telegram_send_text(msg)):
print('訊息傳送成功!')
else:
print('訊息傳送失敗!')
else:
print('無資料')
end=time.time()
print(f'執行時間:{end-start}')
上面 token 與 chat_id 須填入自己的 Telegram 權杖與聊天室識別碼, users 要填入借書證帳號密碼, 執行結果如下 :
>>> %Run ksml_personal_9.py
訊息傳送成功!
Error sending text: httpx.ConnectError:
訊息傳送失敗!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
執行時間:243.84890222549438
除了第一個失敗外 (API 問題) 其餘均傳送成功, 檢視 App 有收到訊息 :
以上是在 PC 上用 Firefox 的 Web driver 做的測試, 要布署到樹莓派或 Mapleboard 上時需要修改為 Chromium 版, 因為在這兩種 Linux 上內建的瀏覽器是 Chromium, 布署版程式如下 :
# ksml_personal_9_deploy.py
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
import re
from datetime import datetime
import time
import requests
import asyncio
from telegram import Bot
async def telegram_send_text(text):
bot=Bot(token=token)
try:
await bot.send_message(
chat_id=chat_id,
text=text
)
return True
except Exception as e:
print(f'Error sending text: {e}')
return False
def get_books(account, password):
try:
# 登入我的書房
options=Options()
options.add_argument("--headless")
driverpath='/usr/lib/chromium-browser/chromedriver'
service=Service(driverpath)
browser=webdriver.Chrome(options=options, service=service)
browser.implicitly_wait(60)
browser.get('https://webpacx.ksml.edu.tw/personal/')
loginid=browser.find_element(By.ID, 'logxinid')
loginid.send_keys(account)
pincode=browser.find_element(By.ID, 'pincode')
pincode.send_keys(password)
div_btn_grp=browser.find_element(By.CLASS_NAME, 'btn_grp')
login_btn=div_btn_grp.find_element(By.TAG_NAME, 'input')
login_btn.click()
# 擷取借閱紀錄
div_redblock=browser.find_element(By.CLASS_NAME, 'redblock')
div_redblock.click()
books=browser.find_elements(By.CLASS_NAME, 'bookdata')
borrow_books=[]
for book in books:
item=dict()
book_name=book.find_element(By.XPATH, './h2/a').text
item['book_name']=book_name.replace('/', '').strip()
reg=r'\d{4}-\d{2}-\d{2}'
due_date=book.find_element(By.XPATH, './ul[4]/li[2]').text
item['due_date']=re.findall(reg, due_date)[0]
due_times=book.find_element(By.XPATH, './ul[5]/li[1]').text
item['due_times']=re.findall(r'\d{1}', due_times)[0]
try:
state=book.find_element(By.XPATH, './ul[6]/li[1]').text
except:
state=''
finally:
if '有人預約' in state:
item['state']=', 有人預約'
else:
item['state']=''
borrow_books.append(item)
browser.back() # 回上一頁
# 擷取預約紀錄
div_blueblock=browser.find_element(By.CLASS_NAME, 'blueblock')
div_blueblock.click()
books=browser.find_elements(By.CLASS_NAME, 'bookdata')
reserve_books=[]
for book in books:
item=dict()
book_name=book.find_element(By.XPATH, './h2/a').text
item['book_name']=book_name.replace('/', '').strip()
sequence=book.find_element(By.XPATH, './ul[7]/li[1]').text
if '預約待取' in sequence: # 已到館
item['ready_for_pickup']=True
reg=r'\d{4}-\d{2}-\d{2}'
item['expiration']=re.findall(reg, sequence)[0]
item['sequence']='0'
else: # 預約中
item['ready_for_pickup']=False
item['expiration']=''
item['sequence']=re.findall(r'\d+', sequence)[0]
reserve_books.append(item)
browser.close()
return (borrow_books, reserve_books)
except Exception as e:
print(e)
return None, None
if __name__ == '__main__':
start=time.time()
token='我的權杖'
chat_id='我的聊天室 ID'
users=[['帳號1', '密碼1'],
['帳號2', '密碼2']]
for user in users:
account=user[0]
password=user[1]
borrow_books, reserve_books=get_books(account, password)
if borrow_books:
borrow=[]
for book in borrow_books:
book_name=book['book_name']
due_times=book['due_times']
due_date=book['due_date']
state=book['state']
due_date=datetime.strptime(due_date, '%Y-%m-%d')
today_str=datetime.today().strftime('%Y-%m-%d')
today=datetime.strptime(today_str, "%Y-%m-%d")
delta=(due_date-today).days
if delta < 0:
msg=f'🅧 {book_name} (逾期 {abs(delta)} 天{state})'
borrow.append(msg)
elif delta == 0:
msg=f'⓿ {book_name} (今日到期, 續借次數 {due_times}{state})'
borrow.append(msg)
elif delta == 1:
msg=f'❶ {book_name} (明日到期, 續借次數 {due_times}{state})'
borrow.append(msg)
elif delta == 2:
msg=f'❷ {book_name} (後天到期, 續借次數 {due_times}{state})'
borrow.append(msg)
elif 2 < delta < 8:
msg=f'✦ {book_name} ({book["due_date"]} 到期, '\
f'續借次數 {due_times}{state})'
borrow.append(msg)
if len(borrow) != 0:
borrow.insert(0, f'\n❖ {account} 的借閱 :')
msg='\n'.join(borrow)
if asyncio.run(telegram_send_text(msg)):
print('訊息傳送成功!')
else:
print('訊息傳送失敗!')
if reserve_books:
reserve=[]
i=0
j=['①', '②', '③', '④', '⑤']
k=['❶', '❷', '❸', '❹', '❺']
for book in reserve_books:
book_name=book['book_name']
sequence=book['sequence']
ready_for_pickup=book['ready_for_pickup']
expiration=book['expiration']
if ready_for_pickup:
msg=f'{k[i]} {book_name} (已到館, 保留期限 {expiration})'
else:
msg=f'{j[i]} {book_name} (順位 {sequence})'
reserve.append(msg)
i += 1
if len(reserve) != 0:
reserve.insert(0, f'\n❖ {account} 的預約 :')
msg='\n'.join(reserve)
if asyncio.run(telegram_send_text(msg)):
print('訊息傳送成功!')
else:
print('訊息傳送失敗!')
else:
print('無資料')
end=time.time()
print(f'執行時間:{end-start}')
注意淡藍底色部分即為使用 Chromium 差異的部分, 其餘與上面 Firefox 完全相同.
用 chmod 指令將此程式檔改為可執行 :
pi@raspberrypi:~ $ sudo chmod +x /home/pi/ksml_personal_9_deploy.py
pi@raspberrypi:~ $ python3 ksml_personal_9_deploy.py
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
訊息傳送成功!
執行時間:573.7787539958954
最後修改 crontab 中的執行程式為 ksml_personal_9_deploy.py 即可 :
pi@raspberrypi:~ $ crontab -e
crontab: installing new crontab
pi@raspberrypi:~ $ crontab -l
0 16 * * 1-5 /usr/bin/python3 /home/pi/twstock_dashboard_update.py
*/31 9-13 * * 1-5 /usr/bin/python3 /home/pi/yahoo_twstock_monitor_table_2.py
0 8,18 * * * /usr/bin/python3 /home/pi/btc_eth_prices_line_notify.py
1 12,17 * * * /usr/bin/python3 /home/pi/technews_3.py
0 9 * * * /usr/bin/python3 /home/pi/books.com.tw_66_telegram.py
0 13,18 * * * /usr/bin/python3 /home/pi/ksml_personal_9_deploy.py
沒有留言 :
張貼留言