[Python] 34장 MVVM 패턴의 웹 크롤링

 

selenium

 

 

MVVM(ajax 사용) 패턴의 경우 나중에 코드를 새로 뿌리기 때문에 크롤링이 불가능함

=> 새로운 방법으로 진행한다.

 

 

 

• 테스트 전 작업

cmd 창에서 아래 코드 기입

 

pip install selenium

 

 

 

• 테스트

my_selenium.py

from selenium import webdriver
from time import sleep
driver = webdriver.Chrome()
url = 'https://www.google.com'
driver.get(url)

sleep(5)

 

결과 화면1 : 실행 후 5초 뒤에 꺼짐

 

 

 

기존 소스로 응용하기

 

 

my_flask.py

from flask import Flask,request,render_template, redirect, jsonify
from flask.helpers import make_response

from day12.daoemp import Daoemp

app = Flask(__name__)

@app.route('/')
def main():
   return redirect("static/emp.html")

@app.route('/axios', methods=['POST'])
def axios():
    data = request.get_json()
    # print(data)
    print(data['menu'])
    return jsonify(message = "ok")

@app.route('/emp_list', methods=['POST'])
def emp_list():
    de = Daoemp()
    list = de.selectList()
    return jsonify(list=list)

@app.route('/emp_one', methods=['POST'])
def emp_one():
    data = request.get_json()
    e_id = data['e_id']
    
    de = Daoemp()
    vo = de.select(e_id);
    return jsonify(vo=vo)

@app.route('/emp_add', methods=['POST'])
def emp_add():
    data = request.get_json()
    e_id = data['e_id']
    e_name = data['e_name']
    gen = data['gen']
    addr = data['addr']
    
    de = Daoemp()
    cnt = de.insert(e_id, e_name, gen, addr);
    return jsonify(cnt=cnt)

@app.route('/emp_mod', methods=['POST'])
def emp_mod():
    data = request.get_json()
    e_id = data['e_id']
    e_name = data['e_name']
    gen = data['gen']
    addr = data['addr']
    
    de = Daoemp()
    cnt = de.update(e_id, e_name, gen, addr);
    return jsonify(cnt=cnt)

@app.route('/emp_del', methods=['POST'])
def emp_del():
    data = request.get_json()
    e_id = data['e_id']
    
    de = Daoemp()
    cnt = de.delete(e_id);
    return jsonify(cnt=cnt)

if __name__ == '__main__':
  app.run(debug=True, port=80, host='0.0.0.0')

 

 

my_selenium.py

from selenium import webdriver
from time import sleep
driver = webdriver.Chrome()
url = 'http://127.0.0.1/static/emp.html'
driver.get(url)

sleep(60)

 

결과 화면2 : 60초 후에 화면 꺼짐

 

 

 

 

• 데이터 값만 출력하기 (find_elemnt 방법)

my_selenium.py

from selenium import webdriver
from time import sleep
from selenium.webdriver.common.by import By
driver = webdriver.Chrome()
url = 'http://127.0.0.1/static/emp.html'
driver.get(url)
# print(driver.page_source)

table = driver.find_element(By.CSS_SELECTOR, "table")

trs = table.find_elements(By.CSS_SELECTOR, "tr")

for idx,tr in enumerate(trs):
    if idx == 0 :
        continue

    tds = tr.find_elements(By.CSS_SELECTOR, "td")
    myname = tds[1].text
    addr = tds[3].text
    print("{}\t{}".format(myname,addr))

 

결과 화면3

 

 

 

• 데이터 값만 출력하기 (select 방법)

my_selenium_bs_select.py

from selenium import webdriver
from time import sleep
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup

driver = webdriver.Chrome()
url = 'http://127.0.0.1/static/emp.html'
driver.get(url)
sleep(1)

html = driver.page_source
soup = BeautifulSoup(html, "lxml")
table = soup.select_one('table')

trs = table.select('tr')
for idx,tr in enumerate(trs) :
    if idx == 0:
        continue
    tds = tr.select('td')

    myname = tds[1].text
    addr = tds[3].text
    print("{}\t{}".format(myname,addr))

sleep(60)

 

결과 화면4

 

 

 

• 데이터 값만 출력하기 (find 방법)

from selenium import webdriver
from time import sleep
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup

driver = webdriver.Chrome()
url = 'http://127.0.0.1/static/emp.html'
driver.get(url)

html = driver.page_source
soup = BeautifulSoup(html, "lxml")
table = soup.find('table')

trs = table.find_all('tr')
for idx,tr in enumerate(trs) :
    if idx == 0:
        continue
    tds = tr.find_all('td')

    myname = tds[1].text
    addr = tds[3].text
    print("{}\t{}".format(myname,addr))

sleep(60)

 

결과 화면5