@@ -14,19 +14,25 @@ npm install context.dev
1414
1515## Usage
1616
17+ Set ` CONTEXT_DEV_API_KEY ` to your API key; the client reads it automatically.
18+
19+ ### Scrape markdown and HTML
20+
1721The full API of this library can be found in [ api.md] ( api.md ) .
1822
1923<!-- prettier-ignore -->
20- ``` js
24+ ``` ts
2125import ContextDev from ' context.dev' ;
2226
23- const client = new ContextDev ({
24- apiKey: process .env [' CONTEXT_DEV_API_KEY' ], // This is the default and can be omitted
25- });
27+ const client = new ContextDev ();
2628
27- const brand = await client .brand .retrieve ({ domain: ' REPLACE_ME' , type: ' by_domain' });
29+ const page = await client .web .scrape ({
30+ url: ' https://example.com' ,
31+ formats: { markdown: true , html: true },
32+ });
2833
29- console .log (brand .request_id );
34+ console .log (page .markdown .data );
35+ console .log (page .html .data );
3036```
3137
3238### Extract structured JSON
@@ -52,19 +58,54 @@ const page = await client.web.scrape({
5258console .log (page .json .data );
5359```
5460
61+ ### Extract relevant highlights
62+
63+ Return the passages that answer a question about the page.
64+
65+ ``` ts
66+ import ContextDev from ' context.dev' ;
67+
68+ const client = new ContextDev ();
69+
70+ const page = await client .web .scrape ({
71+ url: ' https://example.com' ,
72+ formats: { highlights: true },
73+ highlightsParams: { query: ' What is this domain used for?' },
74+ });
75+
76+ console .log (page .highlights .data );
77+ ```
78+
79+ ### Take a screenshot
80+
81+ The screenshot is returned as a base64 image data URL.
82+
83+ ``` ts
84+ import ContextDev from ' context.dev' ;
85+
86+ const client = new ContextDev ();
87+
88+ const page = await client .web .scrape ({
89+ url: ' https://example.com' ,
90+ formats: { screenshot: true },
91+ });
92+
93+ console .log (page .screenshot .data );
94+ ```
95+
5596## What you can do
5697
57- | Task | Method |
58- | ---------------------------------------------------- | ------------------------ |
59- | Scrape a URL to markdown, HTML, JSON or a screenshot | ` client.web.scrape ` |
60- | Crawl a site and get every page as markdown | ` client.web.webCrawlMd ` |
61- | Map every URL on a domain | ` client.web.mapUrls ` |
62- | Search the web | ` client.web.search ` |
63- | Take a screenshot of a page | ` client.web.screenshot ` |
64- | Parse PDFs and documents | ` client.parse.handle ` |
65- | Run thousands of URLs as a batch | ` client.batch.submit ` |
66- | Watch a page for changes | ` client.monitors.create ` |
67- | Look up a company's logo, colors and brand data | ` client.brand.retrieve ` |
98+ | Task | Method |
99+ | ---------------------------------------------------------------- | ------------------------ |
100+ | Scrape a URL to markdown, HTML, JSON, highlights or a screenshot | ` client.web.scrape ` |
101+ | Crawl a site and get every page as markdown | ` client.web.webCrawlMd ` |
102+ | Map every URL on a domain | ` client.web.mapUrls ` |
103+ | Search the web | ` client.web.search ` |
104+ | Take a screenshot of a page | ` client.web.screenshot ` |
105+ | Parse PDFs and documents | ` client.parse.handle ` |
106+ | Run thousands of URLs as a batch | ` client.batch.submit ` |
107+ | Watch a page for changes | ` client.monitors.create ` |
108+ | Look up a company's logo, colors and brand data | ` client.brand.retrieve ` |
68109
69110## Use it from an AI agent
70111
@@ -82,8 +123,8 @@ const client = new ContextDev({
82123 apiKey: process .env [' CONTEXT_DEV_API_KEY' ], // This is the default and can be omitted
83124});
84125
85- const params: ContextDev .BrandRetrieveParams = { domain : ' REPLACE_ME ' , type: ' by_domain ' };
86- const brand : ContextDev .BrandRetrieveResponse = await client .brand . retrieve (params );
126+ const params: ContextDev .WebScrapeParams = { url : ' https://example.com ' , formats: { markdown: true } };
127+ const page : ContextDev .WebScrapeResponse = await client .web . scrape (params );
87128```
88129
89130Documentation for each method, request param, and response field are available in docstrings and will appear on hover in most modern editors.
@@ -96,8 +137,8 @@ a subclass of `APIError` will be thrown:
96137
97138<!-- prettier-ignore -->
98139``` ts
99- const brand = await client .brand
100- .retrieve ({ domain : ' REPLACE_ME ' , type: ' by_domain ' })
140+ const page = await client .web
141+ .scrape ({ url : ' https://example.com ' , formats: { markdown: true } })
101142 .catch (async (err ) => {
102143 if (err instanceof ContextDev .APIError ) {
103144 console .log (err .status ); // 400
@@ -138,7 +179,7 @@ const client = new ContextDev({
138179});
139180
140181// Or, configure per-request:
141- await client .brand . retrieve ({ domain : ' REPLACE_ME ' , type : ' by_domain ' }, {
182+ await client .web . scrape ({ url : ' https://example.com ' , formats : { markdown : true } }, {
142183 maxRetries: 5 ,
143184});
144185```
@@ -155,7 +196,7 @@ const client = new ContextDev({
155196});
156197
157198// Override per-request:
158- await client .brand . retrieve ({ domain : ' REPLACE_ME ' , type: ' by_domain ' }, {
199+ await client .web . scrape ({ url : ' https://example.com ' , formats: { markdown: true } }, {
159200 timeout: 5 * 1000 ,
160201});
161202```
@@ -178,17 +219,17 @@ Unlike `.asResponse()` this method consumes the body, returning once it is parse
178219``` ts
179220const client = new ContextDev ();
180221
181- const response = await client .brand
182- .retrieve ({ domain : ' REPLACE_ME ' , type: ' by_domain ' })
222+ const response = await client .web
223+ .scrape ({ url : ' https://example.com ' , formats: { markdown: true } })
183224 .asResponse ();
184225console .log (response .headers .get (' X-My-Header' ));
185226console .log (response .statusText ); // access the underlying Response object
186227
187- const { data : brand , response : raw } = await client .brand
188- .retrieve ({ domain : ' REPLACE_ME ' , type: ' by_domain ' })
228+ const { data : page , response : raw } = await client .web
229+ .scrape ({ url : ' https://example.com ' , formats: { markdown: true } })
189230 .withResponse ();
190231console .log (raw .headers .get (' X-My-Header' ));
191- console .log (brand . request_id );
232+ console .log (page . markdown . data );
192233```
193234
194235### Logging
@@ -268,8 +309,9 @@ parameter. This library doesn't validate at runtime that the request matches the
268309send will be sent as-is.
269310
270311``` ts
271- client .brand .retrieve ({
272- // ...
312+ client .web .scrape ({
313+ url: ' https://example.com' ,
314+ formats: { markdown: true },
273315 // @ts-expect-error baz is not yet public
274316 baz: ' undocumented option' ,
275317});
0 commit comments