Merge pull request #1096 from getmaxun/pag-limit

fix: llm pagination and limit

server/src/sdk/selectorValidator.ts

@@ -341,35 +341,96 @@ export class SelectorValidator {
 
       if (buttonResult.type && buttonResult.type !== '') {
         if (buttonResult.type === 'clickLoadMore' && buttonResult.selector) {
-          logger.info('Testing Load More button by clicking...');
-          const loadMoreVerified = await this.testLoadMoreButton(buttonResult.selector, listSelector);
+          const selectorExists = await this.verifySelectorExists(buttonResult.selector);
 
-          if (!loadMoreVerified) {
-            logger.warn('Load More button did not load content, falling back to scroll detection');
-            const scrollTestResult = await this.testInfiniteScrollByScrolling(listSelector);
-
-            if (scrollTestResult.detected) {
+          if (!selectorExists) {
+            logger.warn('Load More button selector not found on page, trying fallback next-button detection');
+            const fallback = await this.detectNextButtonFallback(listSelector);
+            if (fallback.type) {
+              logger.info(`Fallback detected pagination type: ${fallback.type}${fallback.selector ? ` with selector: ${fallback.selector}` : ''}`);
               return {
                 success: true,
-                type: 'scrollDown',
-                selector: null
+                type: fallback.type,
+                selector: fallback.selector
               };
             }
           } else {
-            logger.info(`Verified Load More button works`);
+            logger.info('Testing Load More button by clicking...');
+            const loadMoreVerified = await this.testLoadMoreButton(buttonResult.selector, listSelector);
+
+            if (loadMoreVerified) {
+              logger.info(`Verified Load More button works`);
+              return {
+                success: true,
+                type: buttonResult.type,
+                selector: buttonResult.selector
+              };
+            }
+
+            logger.warn('Load More button did not load content, trying fallback next-button detection');
+            const fallback = await this.detectNextButtonFallback(listSelector);
+            if (fallback.type) {
+              logger.info(`Fallback detected pagination type: ${fallback.type}${fallback.selector ? ` with selector: ${fallback.selector}` : ''}`);
+              return {
+                success: true,
+                type: fallback.type,
+                selector: fallback.selector
+              };
+            }
+          }
+
+          logger.warn('Falling back to scroll detection');
+          const scrollTestResult = await this.testInfiniteScrollByScrolling(listSelector);
+
+          if (scrollTestResult.detected) {
             return {
               success: true,
-              type: buttonResult.type,
-              selector: buttonResult.selector
+              type: 'scrollDown',
+              selector: null
+            };
+          }
+        } else if (buttonResult.type === 'scrollDown') {
+          const scrollTestResult = await this.testInfiniteScrollByScrolling(listSelector);
+
+          if (scrollTestResult.detected) {
+            return {
+              success: true,
+              type: 'scrollDown',
+              selector: null
+            };
+          }
+
+          logger.warn('Scroll-based pagination not confirmed, trying fallback next-button detection');
+          const fallback = await this.detectNextButtonFallback(listSelector);
+          if (fallback.type) {
+            logger.info(`Fallback detected pagination type: ${fallback.type}${fallback.selector ? ` with selector: ${fallback.selector}` : ''}`);
+            return {
+              success: true,
+              type: fallback.type,
+              selector: fallback.selector
             };
           }
         } else {
-          logger.info(`Detected pagination type: ${buttonResult.type}${buttonResult.selector ? ` with selector: ${buttonResult.selector}` : ''}`);
-          return {
-            success: true,
-            type: buttonResult.type,
-            selector: buttonResult.selector
-          };
+          if (buttonResult.selector) {
+            const selectorExists = await this.verifySelectorExists(buttonResult.selector);
+            if (!selectorExists) {
+              logger.warn(`Detected pagination selector not found on page, discarding: ${buttonResult.selector}`);
+            } else {
+              logger.info(`Detected pagination type: ${buttonResult.type} with selector: ${buttonResult.selector}`);
+              return {
+                success: true,
+                type: buttonResult.type,
+                selector: buttonResult.selector
+              };
+            }
+          } else {
+            logger.info(`Detected pagination type: ${buttonResult.type}`);
+            return {
+              success: true,
+              type: buttonResult.type,
+              selector: null
+            };
+          }
         }
       }
 
@@ -575,6 +636,132 @@ export class SelectorValidator {
     }
   }
 
+  /**
+   * Verify that a selector (or any of its comma-separated variants) matches at least one element on the page
+   */
+  private async verifySelectorExists(selector: string): Promise<boolean> {
+    if (!this.page || !selector) return false;
+    try {
+      const selectors = selector.split(',').map(s => s.trim()).filter(Boolean);
+      for (const sel of selectors) {
+        const count = await this.page.evaluate((s) => {
+          try {
+            return document.querySelectorAll(s).length;
+          } catch {
+            return 0;
+          }
+        }, sel);
+        if (count > 0) return true;
+      }
+      return false;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Fallback detection of a "next page" style button/link near the list, used when the
+   * primary detection's selector can't be verified or doesn't work as expected
+   */
+  private async detectNextButtonFallback(listSelector: string): Promise<{ type: string; selector: string | null }> {
+    if (!this.page) return { type: '', selector: null };
+
+    try {
+      const result = await this.page.evaluate((listSel) => {
+        const nextTextPatterns = [
+          /^\s*next\s*$/i,
+          /\bnext\s+page\b/i,
+          /^[>\s›→»⟩]+$/,
+          /^>>$/
+        ];
+
+        function isVisible(el: Element): boolean {
+          const rect = el.getBoundingClientRect();
+          if (rect.width === 0 || rect.height === 0) return false;
+          const style = window.getComputedStyle(el);
+          if (style.visibility === 'hidden' || style.display === 'none' || style.opacity === '0') return false;
+          return true;
+        }
+
+        function isNearList(el: Element, listEl: Element | null): boolean {
+          if (!listEl) return true;
+          const elRect = el.getBoundingClientRect();
+          const listRect = listEl.getBoundingClientRect();
+          const verticalGap = Math.max(
+            0,
+            elRect.top - listRect.bottom,
+            listRect.top - elRect.bottom
+          );
+          return verticalGap <= 400;
+        }
+
+        function buildSelector(el: Element): string {
+          const id = el.getAttribute('id');
+          if (id) return `#${CSS.escape(id)}`;
+
+          const rel = el.getAttribute('rel');
+          if (rel === 'next') {
+            return `${el.tagName.toLowerCase()}[rel="next"]`;
+          }
+
+          const ariaLabel = el.getAttribute('aria-label');
+          if (ariaLabel) {
+            return `${el.tagName.toLowerCase()}[aria-label="${ariaLabel}"]`;
+          }
+
+          const classes = Array.from(el.classList).slice(0, 2).map(c => `.${CSS.escape(c)}`).join('');
+          return `${el.tagName.toLowerCase()}${classes}`;
+        }
+
+        let listEl: Element | null = null;
+        try {
+          if (listSel) {
+            const isXPath = listSel.startsWith('//') || listSel.startsWith('(//');
+            if (isXPath) {
+              const xpathResult = document.evaluate(listSel, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+              listEl = xpathResult.singleNodeValue as Element | null;
+            } else {
+              listEl = document.querySelector(listSel);
+            }
+          }
+        } catch {
+          listEl = null;
+        }
+
+        const candidates = Array.from(document.querySelectorAll('a[rel="next"], a, button, [role="button"]'));
+
+        for (const el of candidates) {
+          if (!isVisible(el)) continue;
+
+          const rel = el.getAttribute('rel');
+          if (rel === 'next') {
+            return { type: 'clickNext', selector: buildSelector(el) };
+          }
+
+          const text = (el.textContent || '').trim();
+          const ariaLabel = (el.getAttribute('aria-label') || '').trim();
+
+          const matchesPattern = nextTextPatterns.some(pattern => pattern.test(text) || pattern.test(ariaLabel));
+          if (!matchesPattern) continue;
+
+          if (!isNearList(el, listEl)) continue;
+
+          return { type: 'clickNext', selector: buildSelector(el) };
+        }
+
+        return null;
+      }, listSelector);
+
+      if (result) {
+        return result;
+      }
+      return { type: '', selector: null };
+    } catch (error: any) {
+      logger.error('Error during next-button fallback detection:', error.message);
+      return { type: '', selector: null };
+    }
+  }
+
   /**
    * Clear page reference
    */

server/src/sdk/workflowEnricher.ts

@@ -482,8 +482,9 @@ export class WorkflowEnricher {
       if (objectMatch) jsonStr = objectMatch[0];
 
       const decision = JSON.parse(this.sanitizeJsonString(jsonStr));
-      const parsed = WorkflowEnricher.parseGroupCandidates(decision, elementGroups, decision.limit || null);
-      return WorkflowEnricher.buildDecisionFromCandidates(parsed.candidates, parsed.reasoning, decision.limit || null);
+      const limit = decision.limit || this.extractLimitFromPrompt(prompt) || null;
+      const parsed = WorkflowEnricher.parseGroupCandidates(decision, elementGroups, limit);
+      return WorkflowEnricher.buildDecisionFromCandidates(parsed.candidates, parsed.reasoning, limit);
 
     } catch (error: any) {
       logger.error('LLM decision error:', error);
@@ -491,6 +492,22 @@ export class WorkflowEnricher {
     }
   }
 
+  /**
+   * Extract an item-count limit from a verb-bound number in the prompt (e.g. "scrape 50 products"),
+   * excluding numbers that look like years.
+   */
+  private static extractLimitFromPrompt(prompt: string): number | null {
+    const verbBoundNumber = prompt.match(
+      /\b(?:scrape|extract|get|fetch|pull|grab|collect|need|want|find|return|give\s+me|show\s+me)\s+(\d+)\b/i
+    );
+    if (verbBoundNumber && verbBoundNumber[1]) {
+      const limit = parseInt(verbBoundNumber[1], 10);
+      const isYear = limit >= 1900 && limit <= 2099;
+      if (!isNaN(limit) && limit > 0 && !isYear) return limit;
+    }
+    return null;
+  }
+
   private static extractMeaningfulKeywords(prompt: string): string[] {
     const stopwords = ['extract', 'information', 'from', 'this', 'that', 'these', 'those', 'page', 'website', 'data', 'details', 'given', 'list', 'items', 'show', 'get', 'find', 'all', 'top', 'first', 'last'];
     return prompt.toLowerCase()
@@ -572,9 +589,10 @@ RULES:
 5. Between two viable groups, pick the one with more items AND richer fields (higher tag count, more links/imgs).
 6. If NO group matches at all (login page, error page, irrelevant content), return -1 for both first and second.
 7. NEVER select a group marked [CONTAINS_CODE].
+8. "limit": if the user's request mentions a specific quantity (e.g. "top 50 products", "get 200 results"), set this to that number. Otherwise set it to null.
 
 Reply with ONLY valid JSON. No markdown blocks, no extra text. Pick a best choice AND a runner-up.
-{"first": NUMBER, "second": NUMBER, "reason": "brief explanation"}
+{"first": NUMBER, "second": NUMBER, "reason": "brief explanation", "limit": NUMBER_OR_NULL}
 
 If only one group is viable, set "second" to the same number as "first" (or -1).
 If no group matches, set both to -1.`;
@@ -584,7 +602,7 @@ If no group matches, set both to -1.`;
 GROUPS (format: INDEX: COUNT items |signals| (fields) [flags] - "sample content"):
 ${groupsText}
 
-Pick the BEST group and a RUNNER-UP. Reply JSON: {"first": NUMBER, "second": NUMBER, "reason": "..."}`;
+Pick the BEST group and a RUNNER-UP. Reply JSON: {"first": NUMBER, "second": NUMBER, "reason": "...", "limit": NUMBER_OR_NULL}`;
 
     return { systemPrompt, userPrompt };
   }
@@ -758,7 +776,7 @@ Pick the BEST group and a RUNNER-UP. Reply JSON: {"first": NUMBER, "second": NUM
       selectedGroup: best.group,
       selectedGroupIndex: best.index,
       itemSelector: best.group.xpath,
-      limit: null
+      limit: this.extractLimitFromPrompt(prompt)
     };
   }
 
@@ -1520,6 +1538,36 @@ Return ONLY the list name, nothing else:`;
     }
   }
 
+  /**
+   * Count how many items matching the list selector are currently present on the page
+   */
+  private static async countListItems(
+    listSelector: string,
+    validator: SelectorValidator
+  ): Promise<number> {
+    try {
+      const page = (validator as any).page;
+      if (!page) return 0;
+      const count = await page.evaluate((selector: string) => {
+        try {
+          const isXPath = selector.startsWith('//') || selector.startsWith('(//');
+          if (isXPath) {
+            const result = document.evaluate(
+              selector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null
+            );
+            return result.snapshotLength;
+          }
+          return document.querySelectorAll(selector).length;
+        } catch {
+          return 0;
+        }
+      }, listSelector);
+      return typeof count === 'number' ? count : 0;
+    } catch {
+      return 0;
+    }
+  }
+
   /**
    * Build workflow from LLM decision
    */
@@ -1553,7 +1601,7 @@ Return ONLY the list name, nothing else:`;
       }
 
       logger.info('Extracting field samples and detecting pagination in parallel...');
-      const [fieldSamples, paginationResult] = await Promise.all([
+      const [fieldSamples, paginationResult, itemsOnPage] = await Promise.all([
         this.extractFieldSamples(
           autoDetectResult.fields,
           autoDetectResult.listSelector || '',
@@ -1562,9 +1610,12 @@ Return ONLY the list name, nothing else:`;
         validator.autoDetectPagination(llmDecision.itemSelector).catch((error: any) => {
           logger.warn('Pagination auto-detection failed:', error.message);
           return { success: false, type: 'none', selector: '' };
-        })
+        }),
+        this.countListItems(autoDetectResult.listSelector || llmDecision.itemSelector, validator)
       ]);
 
+      logger.info(`[WorkflowEnricher] Items on current page: ${itemsOnPage}`);
+
       logger.info('Generating semantic field labels with LLM...');
       const fieldLabels = await this.generateFieldLabels(
         autoDetectResult.fields,
@@ -1602,17 +1653,21 @@ Return ONLY the list name, nothing else:`;
         logger.warn(`Low confidence (${filterResult.confidence}) or no fields selected. Using all detected fields as fallback.`);
       }
 
+      const limit = llmDecision.limit || 100;
+      logger.info(`Using limit: ${limit}`);
+
       let paginationType = 'none';
       let paginationSelector = '';
 
-      if (paginationResult.success && paginationResult.type) {
+      const limitFitsOnePage = itemsOnPage > 0 && limit <= itemsOnPage;
+
+      if (limitFitsOnePage) {
+        logger.info(`[WorkflowEnricher] Pagination disabled: requested limit (${limit}) fits within items already on page (${itemsOnPage}).`);
+      } else if (paginationResult.success && paginationResult.type) {
         paginationType = paginationResult.type;
         paginationSelector = paginationResult.selector || '';
       }
 
-      const limit = llmDecision.limit || 100;
-      logger.info(`Using limit: ${limit}`);
-
       logger.info('Generating semantic list name with LLM...');
       const listName = await this.generateListName(
         prompt || 'Extract list data',