如何使用 Oxylabs Scraper API [第 1 部分]：Oxylabs 电子商务 Scraper API

curl --user user:pass1\
'https://data.oxylabs.io/v1/queries' \
-H "内容类型：应用程序/json" \
-d '{"source": "universal_ecommerce", "url": "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html", "callback_url": "https://your.callback.url", "storage_type": "s3", "storage_url": "YOUR_BUCKET_NAME"}'

舶来品 requests
from pprint 舶来品 pprint


# Structure payload.
payload = {
    'source': 'universal_ecommerce',
    'url': 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
    'callback_url': 'https://your.callback.url',
    'storage_type': 's3',
    'storage_url': 'YOUR_BUCKET_NAME'
}

# Get response.
回应 = requests.要求(
    'POST',
    'https://data.oxylabs.io/v1/queries',
    授权=('user', 'pass1'),
    json=payload,
)

# Print prettified response to stdout.
pprint(回应.json())

<?php

$params = 矩阵(
    'source' => 'universal_ecommerce',
    'url' => 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
    'callback_url' => 'https://your.callback.url',
    'storage_type' => 's3',
    'storage_url' => 'YOUR_BUCKET_NAME'
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$headers = 矩阵();
$headers[] = "内容类型：应用程序/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

HTTP method is currently not supported

curl --user user:pass1 'http://data.oxylabs.io/v1/queries/12345678900987654321'

舶来品 requests
from pprint 舶来品 pprint

# Get a response from the stats endpoint.
回应 = requests.要求(
    method=GET,
    网址='http://data.oxylabs.io/v1/queries/12345678900987654321',
    授权=('user', 'pass1'),
)

# Print prettified JSON response to stdout.
pprint(回应.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "http://data.oxylabs.io/v1/queries/12345678900987654321");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

HTTP method is currently not supported

curl --user user:pass1 'http://data.oxylabs.io/v1/queries/12345678900987654321/results'

舶来品 requests
from pprint 舶来品 pprint

# Get response from the stats endpoint.
回应 = requests.要求(
    method=GET,
    网址='http://data.oxylabs.io/v1/queries/12345678900987654321/results',
    授权=('user', 'pass1'),
)

# Print the prettified JSON response to stdout.
pprint(回应.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "http://data.oxylabs.io/v1/queries/12345678900987654321/results");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

HTTP method is currently not supported

# Please see the code samples in Python and PHP.

# This is a simple Sanic web server with a route listening for callbacks on localhost:8080.
# It will print job results to stdout.
舶来品 requests
from pprint 舶来品 pprint
from sanic 舶来品 Sanic, 回应


AUTH_TUPLE = ('user', 'pass1')

app = Sanic()


# Define /job_listener endpoint that accepts POST requests.
@app.route('/job_listener', methods=['POST'])
async def job_listener(要求):
    try:
        res = 要求.json
        links = res.获取('_links', [])
        for 链接 in links:
            如果 链接['rel'] == 'results':
                # Sanic is async, but requests are synchronous, to fully take
                # advantage of Sanic, use aiohttp.
                res_response = requests.要求(
                    method=GET,
                    网址=链接['href'],
                    授权=AUTH_TUPLE,
                )
                pprint(res_response.json())
                break
    except Exception as e:
        打印("Listener exception: {}".格式(e))
    return 回应.json(status=200, 机构={'status': 'ok'})


如果 __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

<?php
$stdout = fopen('php://stdout', 'w');

如果 (isset($_POST)) {
    $result = array_merge($_POST, (矩阵) json_decode(file_get_contents('php://input')));

    $ch = curl_init();

    curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries/".$result['id'].'/results');
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
    curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

    $result = curl_exec($ch);
    fwrite($stdout, $result);

    如果 (curl_errno($ch)) {
        echo 'Error:' . curl_error($ch);
    }
    curl_close ($ch);
}
?>

HTTP method is currently not supported

curl --user user:pass1 'https://data.oxylabs.io/v1/queries/batch' -H 'Content-Type: application/json' \
 -d '@keywords.json'

舶来品 requests
舶来品 json
from pprint 舶来品 pprint


# Get payload from file.
with open('keywords.json', 'r') as f:
    payload = json.loads(f.read())

回应 = requests.要求(
    'POST',
    'https://data.oxylabs.io/v1/queries/batch',
    授权=('user', 'pass1'),
    json=payload,
)

# Print prettified response.
pprint(回应.json())

<?php

$paramsFile = file_get_contents(realpath("keywords.json"));
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries/batch");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $paramsFile);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$headers = 矩阵();
$headers[] = "内容类型：应用程序/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

HTTP method is currently not supported

curl --user user:pass1 'https://data.oxylabs.io/v1/info/callbacker_ips'

舶来品 requests
from pprint 舶来品 pprint

# Get response from the callback IPs endpoint.
回应 = requests.要求(
    method=GET,
    网址='https://data.oxylabs.io/v1/info/callbacker_ips',
    授权=('user', 'pass1'),
)

# Print the prettified JSON response to stdout.
pprint(回应.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/info/callbacker_ips");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

HTTP method is currently not supported

curl --user user:pass1 'https://realtime.oxylabs.io/v1/queries' -H "内容类型：应用程序/json" \
 -d '{"source": "universal_ecommerce", "url": "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html"}'

舶来品 requests
from pprint 舶来品 pprint


# Structure payload.
payload = {
    'source': 'universal_ecommerce',
    'url': 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
}

# Get response.
回应 = requests.要求(
    'POST',
    'https://realtime.oxylabs.io/v1/queries',
    授权=('user', 'pass1'),
    json=payload,
)

# Instead of response with job status and results url, this will return the
# JSON response with results.
pprint(回应.json())

<?php

$params = 矩阵(
    'source' => 'universal_ecommerce',
    'query' => 'sofa',
);

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://realtime.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$headers = 矩阵();
$headers[] = "内容类型：应用程序/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

# URL has to be encoded to escape `&` and `=` characters. It is not necessary in this example.

https://realtime.oxylabs.io/v1/queries?source=universal_ecommerce&url=https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html&access_token=12345abcde

curl -k \
-x realtime.oxylabs.io:60000 \
-U user:pass1 \
-H "X-OxySERPs-User-Agent-Type: desktop_chrome" \
"https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html"

舶来品 requests
from pprint 舶来品 pprint

# Define proxy dict. Do not forget to put your real user and pass here as well.
代理 = {
  'http': 'http://user:[email protected]:60000',
  'https': 'https://user:[email protected]:60000',
}

回应 = requests.要求(
    GET,
    'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
    授权=('user', 'pass1'),
    verify=假的,  # Or accept our certificate.
    代理=代理,
)

# Print result page to stdout
pprint(回应.文本)

# Save returned HTML to result.html file
with open('result.html', 'w') as f:
    f.write(回应.文本)

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'realtime.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, "user" . ":" . "pass1");
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 错误);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 错误);

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

HTTP method is not supported with SuperAPI

curl --user user:pass1 \
'https://data.oxylabs.io/v1/queries' \
-H "内容类型：应用程序/json" \
 -d '{"source": "universal_ecommerce", "url": "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html", "user_agent_type": "mobile", "render": "html", 
 "context": [{"key": "headers", "value": ["Accept-Language": "en-US", "Content-Type": "application/octet-stream", "Custom-Header": "custom header content"]}, {"key": "cookies", "value": [{"key": "NID", "value": "1234567890"}, {"key": "1P JAR", "value": "0987654321"}, {"key": "follow_redirects", "value": true}, {"key": "http_method", "value": "get"}, {"key": "content", "value": "base64EncodedPOSTBody"}, {"key": "successful_status_codes", "value": [303, 808, 909]}]}]}'

舶来品 requests
from pprint 舶来品 pprint


# Structure payload.
payload = {
    'source': 'universal_ecommerce',
    'url': 'https://www.etsy.com/listing/399423455/big-glass-house-planter-handmade-glass?ref=hp_prn&frs=1',
    'user_agent_type': 'desktop',
    'geo_location': 'United States',
    'parse': 真,
    'parser_type': "ecommerce_product",
    'context': [
        {
          'key': 'session_id',
          'value': '1234567890abcdef'
        }
        {
          'key': 'headers', 'value': 
            {
             'Accept-Language': 'en-US',
             'Content-Type': 'application/octet-stream',
             'Custom-Header': 'custom header content'
            }
        },
        {
          'key': 'cookies',
          'value': [{
              'key': 'NID',
             'value': '1234567890'
           },
           {
              'key': '1P_JAR',
             'value': '0987654321'
           }
         ]
        },
        {
          'key': 'follow_redirects',
          'value': 真
        },
        {
          'key': 'successful_status_codes',
          'value': [303, 808, 909]
        },
        {
          'key': 'http_method',
          'value': 'get'
        }
        {
          'key': 'content'
          'value': 'base64EncodedPOSTBody'
        }
    ],
    'callback_url': 'https://your.callback.url',
}

# Get response.
回应 = requests.要求(
    'POST',
    'https://data.oxylabs.io/v1/queries',
    授权=('user', 'pass1'),
    json=payload,
)

# Print prettified response to stdout.
pprint(回应.json())

<?php

$params = [
    'source' => 'universal_ecommerce',
    'url' => 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
    'context' => [
        [
            'key' => 'session_id',
            'value' => '1234567890abcdef'
        ],
        [
            'key' => 'headers',
            'value' => [
                'Accept-Language' => 'en-US',
                'Content-Type' => 'application/octet-stream',
                'Custom-Header' => 'custom header content'
            ],
        ],
        [
            'key' => 'cookies',
            'value' => [
                ['key' => 'NID', 'value' => '1234567890'],
                ['key' => '1P_JAR', 'value' => '0987654321']
            ]
        ],
        [
            'key' => 'follow_redirects',
            'value' => 'true'
        ],
        [
            'key' => 'successful_status_codes',
            'value' => [303, 808, 909]
        ],
        [
            'key' => 'http_method',
            'value' => 'get'
        ],
        [
            'key' => 'content',
            'value' => 'base64EncodedPOSTBody'
        ]
    ]
];

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$headers = 矩阵();
$headers[] = "内容类型：应用程序/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

HTTP method is currently not supported with Push-Pull

curl --user user:pass1 \
'https://data.oxylabs.io/v1/queries' \
-H "内容类型：应用程序/json" \
-d '{"source": "universal_ecommerce", "url": "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html", "user_agent_type": "mobile", "context": [{"key": "headers", "value": ["Accept-Language": "en-US", "Content-Type": "application/octet-stream", "Custom-Header": "custom header content"]}, {"key": "cookies", "value": [{"key": "NID", "value": "1234567890"}, {"key": "1P JAR", "value": "0987654321"}, {"key": "follow_redirects", "value": true}, {"key": "http_method", "value": "get"}, {"key": "content", "value": "base64EncodedPOSTBody"}, {"key": "successful_status_codes", "value": [303, 808, 909]}]}]}'

舶来品 requests
from pprint 舶来品 pprint

# Structure payload.
payload = {
    'source': 'universal_ecommerce',
    'url': 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
    'user_agent_type': 'mobile',
    'geo_location': 'United States',
    'context': [
        {
          'key': 'session_id',
          'value': '1234567890abcdef'
        }
        {
          'key': 'headers', 'value': 
            {
             'Accept-Language': 'en-US',
             'Content-Type': 'application/octet-stream',
             'Custom-Header': 'custom header content'
            }
        },
        {
          'key': 'cookies',
          'value': [{
              'key': 'NID',
             'value': '1234567890'
           },
           {
              'key': '1P_JAR',
             'value': '0987654321'
           }
         ]
        },
        {
          'key': 'follow_redirects',
          'value': 真
        },
        {
          'key': 'successful_status_codes',
          'value': [303, 808, 909]
        },
        {
          'key': 'http_method',
          'value': 'get'
        }
        {
          'key': 'content'
          'value': 'base64EncodedPOSTBody'
        }
    ],
}

# Get response.
回应 = requests.要求(
    'POST',
    'https://realtime.oxylabs.io/v1/queries',
    授权=('user', 'pass1'),
    json=payload,
)

# Instead of response with job status and results url, this will return the
# JSON response with the result.
pprint(回应.json())

<?php

$params = [
    'source' => 'universal_ecommerce',
    'url' => 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
    'context' => [
        [
            'key' => 'session_id',
            'value' => '1234567890abcdef'
        ],
        [
            'key' => 'headers',
            'value' => [
                'Accept-Language' => 'en-US',
                'Content-Type' => 'application/octet-stream',
                'Custom-Header' => 'custom header content'
            ],
        ],
        [
            'key' => 'cookies',
            'value' => [
                ['key' => 'NID', 'value' => '1234567890'],
                ['key' => '1P_JAR', 'value' => '0987654321']
            ]
        ],
        [
            'key' => 'follow_redirects',
            'value' => 'true'
        ],
        [
            'key' => 'successful_status_codes',
            'value' => [303, 808, 909]
        ],
        [
            'key' => 'http_method',
            'value' => 'get'
        ],
        [
            'key' => 'content',
            'value' => 'base64EncodedPOSTBody'
        ]
    ]
];

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v1/queries");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($params));
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$headers = 矩阵();
$headers[] = "内容类型：应用程序/json";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

# The whole string you submit has to be URL-encoded.

https://realtime.oxylabs.io/v1/queries?source=universal_ecommerce&url=https%3A%2F%2Fstackoverflow.com%2Fquestions%2Ftagged%2Fpython&access_token=12345abcde

# A GET request could look something like this:
curl -k \
-x http://realtime.oxylabs.io:60000 \
-U user:pass1 \
"https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html" \
-H "X-OxySERPs-Session-Id: 1234567890abcdef" \
-H "X-OxySERPs-Geo-Location: India" \
-H "Accept-Language: en-US" \
-H "Content-Type: application/octet-stream" \
-H "Custom-Header: custom header content" \
-H "Cookie: NID=1234567890; 1P_JAR=0987654321" \
-H "X-Status-Code: 303, 808, 909"

# A POST request would have the same structure but contain a parameter specifying that it is a POST request:
curl -X POST \
-k \
-x http://realtime.oxylabs.io:60000 \
-U user:pass1 "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html" \
-H "X-OxySERPs-Session-Id: 1234567890abcdef" \
-H "X-OxySERPs-Geo-Location: India" \
-H "Custom-Header: custom header content" \
-H "Cookie: NID=1234567890; 1P_JAR=0987654321" \
-H "X-Status-Code: 303, 808, 909"

舶来品 requests
from pprint 舶来品 pprint

# Define proxy dict. Do not forget to put your real user and pass here as well.
代理 = {
  'http': 'http://user:[email protected]:60000',
  'https': 'https://user:[email protected]:60000',
}

回应 = requests.要求(
    GET,
    'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
    授权=('user', 'pass1'),
    verify=假的,  # Or accept our certificate.
    代理=代理,
)

# Print result page to stdout
pprint(回应.文本)

# Save returned HTML to result.html file
with open('result.html', 'w') as f:
    f.write(回应.文本)

<?php
$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_PROXY, 'realtime.oxylabs.io:60000');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, "user" . ":" . "pass1");
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 错误);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 错误);

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

HTTP method is not supported with SuperAPI

curl --user user:pass1 'https://data.oxylabs.io/v2/stats'

舶来品 requests
from pprint 舶来品 pprint

# Get response from stats endpoint.
回应 = requests.要求(
    method=GET,
    网址='https://data.oxylabs.io/v2/stats',
    授权=('user', 'pass1'),
)

# Print prettified JSON response to stdout.
pprint(回应.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v2/stats");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>

curl --user user:pass1 'https://data.oxylabs.io/v2/stats/limits'

舶来品 requests
from pprint 舶来品 pprint

# Get response from stats endpoint.
回应 = requests.要求(
    method=GET,
    网址='https://data.oxylabs.io/v2/stats/limits',
    授权=('user', 'pass1'),
)

# Print prettified JSON response to stdout.
pprint(回应.json())

<?php

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, "https://data.oxylabs.io/v2/stats/limits");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
curl_setopt($ch, CURLOPT_USERPWD, "user" . ":" . "pass1");

$result = curl_exec($ch);
echo $result;

如果 (curl_errno($ch)) {
    echo 'Error:' . curl_error($ch);
}
curl_close ($ch);
?>