Type Alias DataSourceCrawlConfig

Configuration for webcrawling

type DataSourceCrawlConfig = {
    crawlPatterns?: DataSourceCrawlPattern[];
    indexPatterns?: DataSourceCrawlPattern[];
    scope: DataSourceCrawlScope;
    url: string;
}

Index

Properties

crawlPatterns? indexPatterns? scope url

Properties

`Optional`crawlPatterns

crawlPatterns?: DataSourceCrawlPattern[]

Pattern matching links to crawl

`Optional`indexPatterns

indexPatterns?: DataSourceCrawlPattern[]

Pattern matching links to index

scope

scope: DataSourceCrawlScope

Scope for th webcrawler

url

url: string

Seed URL that crawling will start on.