대전 교통 정보 센터 버스 정보 가져오기

하단의 링크를 이용하여 진행한다.

https://traffic.daejeon.go.kr/bus/busInfoDetail

selenium

MVVM(ajax 사용) 패턴의 경우 나중에 코드를 새로 뿌리기 때문에 크롤링이 불가능함

=> 새로운 방법으로 진행한다.

• 테스트 전 작업

cmd 창에서 아래 코드 기입

pip install selenium

• 테스트

my_selenium.py

from selenium import webdriver
from time import sleep
driver = webdriver.Chrome()
url = 'https://www.google.com'
driver.get(url)

sleep(5)

기존 소스로 응용하기

my_flask.py

from flask import Flask,request,render_template, redirect, jsonify
from flask.helpers import make_response

from day12.daoemp import Daoemp

app = Flask(__name__)

@app.route('/')
def main():
   return redirect("static/emp.html")

@app.route('/axios', methods=['POST'])
def axios():
    data = request.get_json()
    # print(data)
    print(data['menu'])
    return jsonify(message = "ok")

@app.route('/emp_list', methods=['POST'])
def emp_list():
    de = Daoemp()
    list = de.selectList()
    return jsonify(list=list)

@app.route('/emp_one', methods=['POST'])
def emp_one():
    data = request.get_json()
    e_id = data['e_id']
    
    de = Daoemp()
    vo = de.select(e_id);
    return jsonify(vo=vo)

@app.route('/emp_add', methods=['POST'])
def emp_add():
    data = request.get_json()
    e_id = data['e_id']
    e_name = data['e_name']
    gen = data['gen']
    addr = data['addr']
    
    de = Daoemp()
    cnt = de.insert(e_id, e_name, gen, addr);
    return jsonify(cnt=cnt)

@app.route('/emp_mod', methods=['POST'])
def emp_mod():
    data = request.get_json()
    e_id = data['e_id']
    e_name = data['e_name']
    gen = data['gen']
    addr = data['addr']
    
    de = Daoemp()
    cnt = de.update(e_id, e_name, gen, addr);
    return jsonify(cnt=cnt)

@app.route('/emp_del', methods=['POST'])
def emp_del():
    data = request.get_json()
    e_id = data['e_id']
    
    de = Daoemp()
    cnt = de.delete(e_id);
    return jsonify(cnt=cnt)

if __name__ == '__main__':
  app.run(debug=True, port=80, host='0.0.0.0')

my_selenium.py

from selenium import webdriver
from time import sleep
driver = webdriver.Chrome()
url = 'http://127.0.0.1/static/emp.html'
driver.get(url)

sleep(60)

• 데이터 값만 출력하기 (find_elemnt 방법)

my_selenium.py

from selenium import webdriver
from time import sleep
from selenium.webdriver.common.by import By
driver = webdriver.Chrome()
url = 'http://127.0.0.1/static/emp.html'
driver.get(url)
# print(driver.page_source)

table = driver.find_element(By.CSS_SELECTOR, "table")

trs = table.find_elements(By.CSS_SELECTOR, "tr")

for idx,tr in enumerate(trs):
    if idx == 0 :
        continue

    tds = tr.find_elements(By.CSS_SELECTOR, "td")
    myname = tds[1].text
    addr = tds[3].text
    print("{}\t{}".format(myname,addr))

• 데이터 값만 출력하기 (select 방법)

my_selenium_bs_select.py

from selenium import webdriver
from time import sleep
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup

driver = webdriver.Chrome()
url = 'http://127.0.0.1/static/emp.html'
driver.get(url)
sleep(1)

html = driver.page_source
soup = BeautifulSoup(html, "lxml")
table = soup.select_one('table')

trs = table.select('tr')
for idx,tr in enumerate(trs) :
    if idx == 0:
        continue
    tds = tr.select('td')

    myname = tds[1].text
    addr = tds[3].text
    print("{}\t{}".format(myname,addr))

sleep(60)

• 데이터 값만 출력하기 (find 방법)

from selenium import webdriver
from time import sleep
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup

driver = webdriver.Chrome()
url = 'http://127.0.0.1/static/emp.html'
driver.get(url)

html = driver.page_source
soup = BeautifulSoup(html, "lxml")
table = soup.find('table')

trs = table.find_all('tr')
for idx,tr in enumerate(trs) :
    if idx == 0:
        continue
    tds = tr.find_all('td')

    myname = tds[1].text
    addr = tds[3].text
    print("{}\t{}".format(myname,addr))

sleep(60)

매일 경제 전종목 게시판

daostock.py

import pymysql

class DaoStock:
    def __init__(self):
        self.con = pymysql.connect(host='localhost', port=3305, user='root',
                              password='python', db='python', charset='utf8')

        self.cur = self.con.cursor(pymysql.cursors.DictCursor)

    def insert(self,s_code,s_name,price,ymd):
        sql = f"""
            INSERT INTO 
                stock (s_code,s_name,price,ymd)
            VALUES ("{s_code}","{s_name}","{price}","{ymd}")
        """
        cnt = self.cur.execute(sql)
        self.con.commit()
        return cnt
        
    def __del__(self):
        self.cur.close()
        self.con.close()
        
if __name__ == '__main__':
    de = DaoStock()
    cnt = de.insert('1', '1', '1', '1')
    print("cnt",cnt)

mybs01.py

import requests, datetime
from bs4 import BeautifulSoup
from day18.daostock import DaoStock

ds = DaoStock()
ymd = datetime.datetime.today().strftime('%Y%m%d_%H%M')
print("ymd",ymd)

res = requests.get('https://stock.mk.co.kr/domestic/all_stocks')

soup = BeautifulSoup(res.text, "lxml")
divs = soup.find_all("div", {'class':"st_name"})

for idx,div in enumerate(divs):
    s_name = div.text.strip()
    s_code = div.find("a")['href'].split("/")[3]
    price = div.parent.find_all("div")[1].text.strip().replace(",","")
    cnt = ds.insert(s_code, s_name, price, ymd)
    print(idx,s_name,s_code,price,ymd,cnt)