Unable to get dynamic elements after scrolling

[rsinghtc]

Here's my code :

LET doc = DOCUMENT('https://www.vrbo.com/search/keywords:atlanta-georgia-united-states-of-america/arrival:2020-09-01/departure:2020-09-02?adultsCount=1&petIncluded=false', { driver: "cdp" })
WAIT_ELEMENT(doc, '.media-flex__content', 50000)

LET result = (
FOR page IN PAGINATION(doc, '[data-wdio="ResultsFooter"]')
LIMIT 20
SCROLL_ELEMENT(doc, '[data-wdio="ResultsFooter"]')
WAIT(500)
RETURN ""
)

LET listings = ELEMENTS(doc, '.media-flex__content')
FOR listing IN listings
LET title = ELEMENT(listing, '.HitInfo__headline')
LET Price_per_night = ELEMENT(listing, '.DualPrice__amount')
RETURN {
title: title.innerText,
price_per_night: Price_per_night.innerText
}

Ideally i am expecting to scrape all the 50 elements on the page , but what's happening is it gives me top 2 results and the last one . How do I tell ferret to lazy scroll and then scrape ? Is this also case of infinite scroll ?

-Rahul

[ziflex]

Hey, good news! WHILE loop has landed master branch! You may try this query:

LET doc = DOCUMENT("https://www.vrbo.com/search/keywords:atlanta-georgia-united-states-of-america?adultsCount=1&petIncluded=false", { driver: "cdp" })

LET canDoNext = 'a[data-wdio="pager-next"]:not(:disabled)'
LET nextBtnSelector = '.Pager__li--next'
LET loaderSelector = '.HitCollection--placeholder'

WAIT_NO_ELEMENT(doc, loaderSelector)

FOR p DO WHILE ELEMENT_EXISTS(doc, canDoNext)
	LET wait = p > 0 ? CLICK(doc, nextBtnSelector) && WAIT_NO_ELEMENT(doc, loaderSelector): false
	LET pageSize = ELEMENTS_COUNT(doc, '[data-wdio="HitCollection"] > div')
	
	FOR i IN 1..pageSize
		LET selector = FMT('[data-wdio="Waypoint{}"]', i)
		SCROLL_ELEMENT(doc, selector)
		WAIT(100)

		RETURN {
			title: INNER_TEXT(doc, FMT("{} .HitInfo__headline", selector))
		}