Add selenium to tsv example from bob. - brcon2023-hackathons - Bitreichcon 2023 Hackathon Repository (HTM) git clone git://bitreich.org/brcon2023-hackathons git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/brcon2023-hackathons (DIR) Log (DIR) Files (DIR) Refs (DIR) Tags --- (DIR) commit d2f3f8bf36e6d7b0d88f7d3e02353bcd87a93795 (DIR) parent a7cd0c547c792f74b7784cc0a8c806380a28ca2f (HTM) Author: Christoph Lohmann <20h@r-36.net> Date: Thu, 10 Aug 2023 16:14:57 +0200 Add selenium to tsv example from bob. Diffstat: A sfeed-atom/selenium_crawl_tsv.py | 118 +++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+), 0 deletions(-) --- (DIR) diff --git a/sfeed-atom/selenium_crawl_tsv.py b/sfeed-atom/selenium_crawl_tsv.py @@ -0,0 +1,118 @@ +from selenium import webdriver +from selenium.webdriver.common.by import By + +from selenium.webdriver.firefox.options import Options +from selenium.webdriver.firefox.firefox_profile import FirefoxProfile + +import sys +from datetime import datetime + +def make_escape_content_trans(): + m = {} + for i in range(0, 32): + m[i] = "" + m[0x7f] = "" # DEL + # replace + m["\\"] = "\\\\" + m["\n"] = "\\n" + m["\t"] = "\\t" + + return str.maketrans(m) + +def make_escape_field_trans(): + m = {} + for i in range(0, 32): + m[i] = "" + m[0x7f] = "" # DEL + # replace + m["\n"] = " " + m["\t"] = " " + + return str.maketrans(m) + +escape_content_tbl = make_escape_content_trans() +escape_field_tbl = make_escape_field_trans() + +def escape_content(s): + return s.translate(escape_content_tbl).strip() + +def escape_field(s): + return s.translate(escape_field_tbl).strip() + +if len(sys.argv) > 1: + url = sys.argv[1] +else: + print("usage: <url>") + sys.exit(1) + +options = Options() +options.add_argument("--headless") + +# use existing profile: + +#options.add_argument("--profile") +#profile_path = "/home/hiltjo/.mozilla/firefox/z86g7oxr.default-release" +# NOTE: must not be running at the same time. +#options.add_argument(profile_path) +#options.set_preference("profile", profile_path) + +# setup custom profile: +# JS disabled +options.set_preference("javascript.enabled", False) +# disable stylesheet +options.set_preference("permissions.default.stylesheet", 2) +# disable image loading +options.set_preference("permissions.default.image", 2) +# override user-agent. +#options.set_preference("general.useragent.override", "whatever you want") + +driver = webdriver.Firefox(options=options) + +# set timeouts +#driver.implicitly_wait(10) + +# get the page +driver.get(url) + +# print page title +#print(driver.title) + +#pagesource = driver.execute_script("return document.body.InnerHTML;") +#print(pagesource) +#print(driver.page_source) +#outer_html = driver.find_element(By.XPATH, "//body").get_attribute("outerHTML") + +#outer_html = driver.find_element(By.TAG_NAME, "html").get_attribute("outerHTML") +#print(outer_html) + +# show all links on a page +#links = driver.find_elements(By.TAG_NAME, "a") +anchors = driver.find_elements(By.CSS_SELECTOR, "main a") +links = [] +for anchor in anchors: + href = anchor.get_attribute("href") + text = anchor.text + if len(href): + links.append({"href": href, "text": text}) + +for link in links: + driver.get(link["href"]) + + # parse timestamp. + time = driver.find_element(By.TAG_NAME, "time") + ts = datetime.strptime(time.text, "%Y-%m-%d") + ts = int(ts.timestamp()) + + content = driver.find_element(By.CSS_SELECTOR, "article").get_attribute("outerHTML") + title = driver.title + title = title.replace(" - Codemadness", "") + + # escape fields + content = escape_content(content) + title = escape_field(title) + link = escape_field(link["href"]) + + print("%d\t%s\t%s\t%s\thtml" % (ts, title, link, content)) + +driver.close() +driver.quit()