Feature/eval template (#651)

* Refactored use of eval

* Disable unstable unit test

e2e/cli.go

@@ -9,6 +9,7 @@ import (
 	"github.com/rs/zerolog"
 	"io/ioutil"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"strings"
 
@@ -136,10 +137,23 @@ func main() {
 		runtime.WithLogLevel(logging.MustParseLevel(*logLevel)),
 	}
 
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	signal.Notify(c, os.Kill)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	go func() {
+		for {
+			<-c
+			cancel()
+		}
+	}()
+
 	if query != "" {
-		err = execQuery(engine, opts, query)
+		err = execQuery(ctx, engine, opts, query)
 	} else {
-		err = execFiles(engine, opts, files)
+		err = execFiles(ctx, engine, opts, files)
 	}
 
 	if err != nil {
@@ -148,7 +162,7 @@ func main() {
 	}
 }
 
-func execFiles(engine *ferret.Instance, opts []runtime.Option, files []string) error {
+func execFiles(ctx context.Context, engine *ferret.Instance, opts []runtime.Option, files []string) error {
 	errList := make([]error, 0, len(files))
 
 	for _, path := range files {
@@ -187,7 +201,7 @@ func execFiles(engine *ferret.Instance, opts []runtime.Option, files []string) e
 			}
 
 			if len(dirFiles) > 0 {
-				if err := execFiles(engine, opts, dirFiles); err != nil {
+				if err := execFiles(ctx, engine, opts, dirFiles); err != nil {
 					log.Debug().Err(err).Msg("failed to execute files")
 
 					errList = append(errList, err)
@@ -214,7 +228,7 @@ func execFiles(engine *ferret.Instance, opts []runtime.Option, files []string) e
 
 		log.Debug().Msg("successfully read file")
 		log.Debug().Msg("executing file...")
-		err = execQuery(engine, opts, string(out))
+		err = execQuery(ctx, engine, opts, string(out))
 
 		if err != nil {
 			log.Debug().Err(err).Msg("failed to execute file")
@@ -239,8 +253,8 @@ func execFiles(engine *ferret.Instance, opts []runtime.Option, files []string) e
 	return nil
 }
 
-func execQuery(engine *ferret.Instance, opts []runtime.Option, query string) error {
-	out, err := engine.Exec(context.Background(), query, opts...)
+func execQuery(ctx context.Context, engine *ferret.Instance, opts []runtime.Option, query string) error {
+	out, err := engine.Exec(ctx, query, opts...)
 
 	if err != nil {
 		return err

e2e/tests/dynamic/doc/wait/attr.fql

@@ -11,11 +11,10 @@ LET prev = el.attributes
 
 ATTR_SET(el, attrName, attrVal)
 WAIT_ATTR(doc, selector, attrName, attrVal, 30000)
-//WAIT_ATTR(el, attrName, attrVal)
 
 LET curr = el.attributes
 
 T::NONE(prev[attrName])
 T::EQ(attrVal, curr[attrName], "attributes should be updated")
 
-RETURN NONE
+RETURN TRUE

e2e/tests/dynamic/doc/wait/attr_all.fql

@@ -6,4 +6,4 @@ WAIT_ELEMENT(doc, "#page-events")
 CLICK_ALL(doc, "#wait-class-btn, #wait-class-random-btn")
 WAIT_ATTR_ALL(doc, "#wait-class-content, #wait-class-random-content", "class", "alert alert-success", 10000)
 
-RETURN NONE
+RETURN TRUE

e2e/tests/dynamic/element/attrs/get_2.fql

@@ -8,8 +8,6 @@ WAIT_ELEMENT(doc, pageSelector)
 LET el = ELEMENT(doc, elemSelector)
 LET attrs = ATTR_GET(el, "style")
 
-T::EQ(attrs.style, {
-    display: "block"
-})
+T::EQ(attrs.style.display, "block")
 
-RETURN NONE
+RETURN TRUE

e2e/tests/dynamic/element/attrs/remove.fql

@@ -2,19 +2,19 @@ LET url = @lab.cdn.dynamic + "?redirect=/events"
 LET doc = DOCUMENT(url, true)
 LET pageSelector = "#page-events"
 LET elemSelector = "#wait-no-style-content"
+LET attrName = "data-e2e-test"
 
 WAIT_ELEMENT(doc, pageSelector)
 LET el = ELEMENT(doc, elemSelector)
+ATTR_SET(el, attrName, "true")
 
-LET prev = el.attributes.style
+LET prev = el.attributes[attrName]
 
-ATTR_REMOVE(el, "style")
+ATTR_REMOVE(el, attrName)
 
-LET curr = el.attributes.style
+LET curr = el.attributes[attrName]
 
-T::EQ(prev, {
-    display: "block"
-})
+T::EQ(prev, "true")
 T::NONE(curr)
 
-RETURN NONE
+RETURN TRUE

e2e/tests/dynamic/element/attrs/remove_style.fql

@@ -0,0 +1,27 @@
+LET url = @lab.cdn.dynamic + "?redirect=/events"
+LET doc = DOCUMENT(url, true)
+LET pageSelector = "#page-events"
+LET elemSelector = "#wait-no-style-content"
+LET styleName = "color"
+LET styleValue = "rgb(100, 100, 100)"
+
+WAIT_ELEMENT(doc, pageSelector)
+LET el = ELEMENT(doc, elemSelector)
+
+LET prev = el.style.color
+
+T::NOT::EQ(prev, styleValue)
+
+STYLE_SET(el, styleName, styleValue)
+
+LET curr = el.style.color
+
+T::EQ(curr, styleValue)
+
+ATTR_REMOVE(el, "style")
+
+LET removed = el.style.color
+
+T::EQ(prev, removed)
+
+RETURN TRUE

e2e/tests/dynamic/element/attrs/set.fql

@@ -2,22 +2,21 @@ LET url = @lab.cdn.dynamic + "?redirect=/events"
 LET doc = DOCUMENT(url, true)
 LET pageSelector = "#page-events"
 LET elemSelector = "#wait-no-style-content"
+LET attrName = "data-e2e-test"
 
 WAIT_ELEMENT(doc, pageSelector)
 
 LET el = ELEMENT(doc, elemSelector)
-LET prev = el.attributes.style
+ATTR_SET(el, attrName, "true")
 
-ATTR_SET(el, "style", {
-    color: "black"
-})
+LET prev = el.attributes[attrName]
 
-WAIT(200)
+T::EQ(prev, "true")
 
-LET curr = el.attributes.style
+ATTR_SET(el, attrName, "false")
 
-PRINT(el.attributes.style)
+LET curr = el.attributes[attrName]
 
-T::EQ(curr.color, "black")
+T::EQ(curr, "false")
 
-RETURN NONE
+RETURN TRUE

e2e/tests/dynamic/element/attrs/set_many.fql

@@ -2,17 +2,18 @@ LET url = @lab.cdn.dynamic + "?redirect=/events"
 LET doc = DOCUMENT(url, true)
 LET pageSelector = "#page-events"
 LET elemSelector = "#wait-no-style-content"
+LET color = "rgb(66, 66, 66)"
 
 WAIT_ELEMENT(doc, pageSelector)
 
 LET el = ELEMENT(doc, elemSelector)
 
 ATTR_SET(el, {
-    style: "color: black;",
+    style: "color:" + color,
     "data-ferret-x": "test"
 })
 
-T::EQ(el.attributes.style.color, "black")
+T::EQ(el.attributes.style.color, color)
 T::EQ(el.attributes["data-ferret-x"], "test")
 
-RETURN NONE
+RETURN TRUE

e2e/tests/dynamic/element/styles/remove.fql

@@ -10,11 +10,9 @@ LET prev = el.attributes.style
 
 STYLE_REMOVE(el, "display")
 
-WAIT(1000)
-
 LET curr = el.attributes.style
 
 T::EQ(prev.display, "block")
-T::NONE(curr.display)
+T::EQ(curr.display, "inline")
 
-RETURN NONE
+RETURN TRUE

e2e/tests/dynamic/element/wait/attr_2.fql

@@ -9,6 +9,7 @@ ATTR_SET(el, "data-test", "test")
 WAIT_ATTR(el, "data-test", "test")
 
 ATTR_REMOVE(el, "class")
+
 WAIT_ATTR(el, "class", NONE)
 
 T::NONE(el.attributes.class, "attribute should be removed")

e2e/tests/examples/pagination.yaml

@@ -5,4 +5,4 @@ query:
     criteria: "scraper"
     pages: 2
 assert:
-  text: RETURN T::NOT::EMPTY(@lab.data.query.result)
+  text: RETURN T::NOT::NONE(@lab.data.query.result)

examples/pagination.fql

@@ -1,29 +1,67 @@
 LET baseURL = 'https://www.amazon.com/'
 LET amazon = DOCUMENT(baseURL, { driver: "cdp" })
 
+WAIT_ELEMENT(amazon, '#nav-search-submit-button')
 INPUT(amazon, '#twotabsearchtextbox', @criteria)
 CLICK(amazon, '#nav-search-submit-button')
-WAIT_NAVIGATION(amazon)
 
-LET resultListSelector = '[data-component-type="s-search-results"]'
-LET resultItemSelector = '[data-component-type="s-search-result"]'
-LET nextBtnSelector = '.s-pagination-next:not(.s-pagination-disabled)'
+WAITFOR EVENT "navigation" IN amazon OPTIONS { target: "www\.amazon\.com\/s/ref"} 50000
+WAIT_ELEMENT(amazon, '[class*="template=PAGINATION"]')
+
+LET paginator = ELEMENT(amazon, '[class*="-pagination"]')
+LET foundPrefixes = (FOR cn IN SPLIT(paginator.attributes.class, " ")
+    FILTER cn LIKE "*-pagination*"
+    LIMIT 1
+    RETURN FIRST(SPLIT(cn, "-"))
+)
+
+LET prefix = FIRST(foundPrefixes)
+T::NOT::EMPTY(prefix, "CSS prefix should not be empty")
+PRINT("CSS Prefix is:", prefix)
+
+LET paginationItems = paginator.length
+
+LET variants = {
+    "s": {
+        nextBtnSelector: ".s-pagination-next",
+        pagersSelector: ".s-pagination-item:not(.s-pagination-next, .s-pagination-previous):last-of-type"
+    },
+    "a": {
+        nextBtnSelector: ".a-pagination .a-last",
+        pagersSelector: FMT("ul.a-pagination li:nth-of-type({})", paginator.length - 1)
+    }
+}
+
+LET selectors = variants[prefix]
+
+T::NOT::NONE(selectors, "Supported CSS selectors not found")
+
+LET spinner = FMT('[data-component-type="{0}-search-results"] .{0}-result-list-placeholder', prefix)
+LET resultListSelector = FMT('[data-component-type="{}-search-results"]', prefix)
+LET resultItemSelector = FMT('[data-component-type="{}-search-result"]', prefix)
+
+LET pagersSelector = FMT('.{0}-pagination :not(.{0}-last)', prefix)
 LET priceWholeSelector = '.a-price-whole'
 LET priceFracSelector = '.a-price-fraction'
-LET pagers = ELEMENTS(amazon, '.s-pagination-item.s-pagination-disabled')
+
+LET pagers = ELEMENTS(amazon, pagersSelector)
 LET pages = LENGTH(pagers) > 0 ? TO_INT(INNER_TEXT(LAST(pagers))) : 0
 
+PRINT("Found pages:", pages)
+
 LET result = (
     FOR pageNum IN 1..pages
         LIMIT @pages
 
-        LET clicked = pageNum == 1 ? false : CLICK(amazon, nextBtnSelector)
-        LET waitSelector = clicked ? WAIT_ELEMENT(amazon, resultListSelector) : false
+        LET clicked = pageNum == 1 ? false : CLICK(amazon, selectors.nextBtnSelector)
+        LET waitSelector = clicked ? WAIT_NO_CLASS(amazon, spinner, 'aok-hidden') && WAIT_ELEMENT(amazon, resultItemSelector) : false
 
         PRINT("page:", pageNum, "clicked", clicked)
 
+        LET found = ELEMENTS(amazon, resultItemSelector)
+
         LET items = (
-            FOR el IN ELEMENTS(amazon, resultItemSelector)
+            FOR el IN found
                 LET hasPrice = ELEMENT_EXISTS(el, priceWholeSelector)
                 LET priceWholeTxt = hasPrice ? FIRST(REGEX_MATCH(INNER_TEXT(el, priceWholeSelector), "[0-9]+")) : "0"
                 LET priceFracTxt = hasPrice ? FIRST(REGEX_MATCH(INNER_TEXT(el, priceFracSelector), "[0-9]+")) : "00"
@@ -42,3 +80,5 @@ LET result = (
 )
 
 RETURN FLATTEN(result)
+
+