From 2afd36584b4ef3420e4145983d827f19c53fd6be Mon Sep 17 00:00:00 2001 From: Julien Riou Date: Sun, 28 Feb 2021 15:11:31 +0100 Subject: [PATCH] Add store support for Micro Center (#2) Local stores are set with an "storeid" param in the query string of the URL and by a "storeSelected" cookie to avoid garbage in the query strings for further requests. As product URL is a unique key in the database, Micro Center is able to handle an URL with the store ID for every product. We can add this storeid in the list of URLs to parse and job done. Every single Micro Center local store are parsable. Signed-off-by: Julien Riou --- parser.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/parser.go b/parser.go index 2bbddff..5065b31 100644 --- a/parser.go +++ b/parser.go @@ -342,6 +342,11 @@ LET first_page = '` + url + `' LET doc = DOCUMENT(first_page, {driver: "cdp"}) LET base_url = 'https://www.microcenter.com' +// determine store id or default to "shippable items" (online store) +LET detected_store_id = LAST(REGEX_MATCH(first_page, '&storeid=([0-9]*)', true)) +LET store_id = detected_store_id ? detected_store_id : "029" // id of the online store if not found +COOKIE_SET(doc, {"name": "storeSelected", "value": store_id}) + LET next_pages = ( FOR a IN ELEMENTS(doc, "div .pagination .pages a") RETURN base_url + a.attributes.href @@ -361,7 +366,7 @@ LET results = ( LET available = LENGTH(ELEMENTS(el, "div .price_wrapper form .STBTN"))>0 RETURN { name: name, - url: base_url + url, + url: base_url + url + "?storeid=" + store_id, price: price, price_currency: "USD", available: available,