import requests, bs4, urllib, sys, re, math url = str(sys.argv[1]); query = urllib.parse.quote_plus(url) headers = { 'Referer': 'http://www.bipum.net/', 'Save-Data': 'on', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36' } resp = requests.get(url, headers = headers) resp.raise_for_status() resp.encoding='UTF-8' html = resp.text bs = bs4.BeautifulSoup(html, 'html.parser') prdcodeSrc = bs.select('div.tbl_info td.code_view div p') if len(prdcodeSrc) == 0 : prdCode = "none" else : prdCode = bs.select('div.tbl_info td.code_view div p')[0].getText().strip() print(url + "\t" + prdCode)