Pagination
An example of how to use pagination for fetching a lot of data at once from the REST API.
If your application needs to fetch more data than a single request can handle, pagination can be used to seamlessly fetch blocks of the data, or pages, until completion. It splits a large request into smaller ones at the cost of a few extra lines of code. While this example shows how one can fetch projects with pagination, it can be adapted for other requests if desired.
- Basic Auth For simplicity, we here use Basic Auth for authentication. We recommend replacing this with an OAuth2 flow production-level integrations.
- Service Account Credentials You must create and know the credentials of a Service Account. Any role will suffice.
- REST API This example utilizes our REST API to interact with our cloud. See the REST API Reference for a full list of available endpoints.
The following points summarize the provided example code.
- Projects are fetched in groups of 100 at a time.
- When all projects have been fetched, print the total number of projects.
- Iterate through and print the display name for each project.
If you wish to run the code locally, make sure you have a working runtime environment.
Python 3.10
Python API
Node.js 17
Go 1.17
The following packages are required to run the example code.
pip install requests==2.27.1
pip install --upgrade disruptive
The following modules are required by the example code and must be installed.
npm install [email protected]
No additional packages are required.
Add the following environment variables as they will be used to authenticate the API.
Bash
export DT_SERVICE_ACCOUNT_KEY_ID=<YOUR_SERVICE_ACCOUNT_KEY_ID>
export DT_SERVICE_ACCOUNT_SECRET=<YOUR_SERVICE_ACCOUNT_SECRET>
export DT_SERVICE_ACCOUNT_EMAIL=<YOUR_SERVICE_ACCOUNT_EMAIL>
The following code snippet implements pagination in a few languages.
Python 3.10
Python API
Node.js 17
Go 1.17
import os
import requests
# Authentication credentials loaded from environment.
SERVICE_ACCOUNT_KEY_ID = os.getenv('DT_SERVICE_ACCOUNT_KEY_ID', '')
SERVICE_ACCOUNT_SECRET = os.getenv('DT_SERVICE_ACCOUNT_SECRET', '')
# Shortform Disruptive REST API base url.
BASE_URL = 'https://api.d21s.com/v2'
def paginated_get(url: str, result_field: str, key_id: str, secret: str):
results = []
# Create a parameters dictionary that contains an empty page token.
params: dict = {'pageToken': ''}
# Loop until all pages have been fetched.
print('Paging...')
while True:
# Send GET request for projects.
page = requests.get(url, params=params, auth=(key_id, secret)).json()
if 'error' in page:
raise Exception(page)
elif result_field not in page:
raise KeyError(f'Field "{result_field}" not in response.')
# Concatenate the request response to output.
results += page[result_field]
print(f'- Got {len(page[result_field])} results in page.')
# Update parameters dictionary with next page token.
if len(page['nextPageToken']) > 0:
params['pageToken'] = page['nextPageToken']
else:
break
return results
if __name__ == '__main__':
# Make paginated requests for all available projects.
projects = paginated_get(
url=BASE_URL + '/projects',
result_field='projects',
key_id=SERVICE_ACCOUNT_KEY_ID,
secret=SERVICE_ACCOUNT_SECRET,
)
# Print display name of all fetched projects.
for i, project in enumerate(projects):
print('{}. {}'.format(i, project['displayName']))
The Python API does automatic pagination and requires no additional code.
import disruptive as dt
if __name__ == '__main__':
# Make paginated requests for all available projects.
projects = dt.Project.list_projects()
# Print display name of all fetched projects.
for i, project in enumerate(projects):
print('{}. {}'.format(i, project.display_name))
const axios = require('axios').default
// Environment variables for authentication and target.
const serviceAccountKeyId = process.env.DT_SERVICE_ACCOUNT_KEY_ID
const serviceAccountSecret = process.env.DT_SERVICE_ACCOUNT_SECRET
// Shortform Disruptive REST API base url.
const baseUrl = 'https://api.d21s.com/v2'
async function paginatedGet(url, resultField, keyId, secret) {
let results = []
// Create a parameters dictionary that contains an empty page token.
let params = {'pageToken': ''}
// Loop until all pages have been fetched.
console.log('Paging...')
while (true) {
// Send GET request for projects.
let page = await axios({
method: 'GET',
url: url,
auth: {
username: keyId,
password: secret,
},
params: params,
}).catch(function (error) {
if (error.response) {
throw new Error(
error.response.data.code + ' - '
+ error.response.data.error
)
}
})
if (!(resultField in page.data)) {
throw new Error('Field "' + resultField + '" not in response.')
}
// Concatenate response contents to output list.
results.push(...page.data[resultField])
console.log(`- ${page.data[resultField].length} projects in page.`)
// Update parameters with next page token.
if (page.data.nextPageToken.length > 0) {
params.pageToken = page.data.nextPageToken
} else {
break
}
}
return results
}
async function main () {
// Make paginated requests for all available projects.
let projects = []
projects = await paginatedGet(
baseUrl + '/projects',
'projects',
serviceAccountKeyId,
serviceAccountSecret,
)
// Print display name of all fetched projects.
for (let i = 0; i < projects.length; i++) {
console.log(`${i}. ${projects[i]['displayName']}`)
}
}
main().catch((err) => {console.log(err)});
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"time"
)
const (
baseURL = "https://api.d21s.com/v2"
)
// Read the environment variables used for authentication
var keyID = os.Getenv("DT_SERVICE_ACCOUNT_KEY_ID")
var secret = os.Getenv("DT_SERVICE_ACCOUNT_SECRET")
// Struct that represents a project fetched from the API
type Project struct {
Name string `json:"name"`
DisplayName string `json:"displayName"`
Inventory bool `json:"inventory"`
Organization string `json:"organization"`
OrganizationDisplayName string `json:"organizationDisplayName"`
Sensorcount int `json:"sensorCount"`
CloudConnectorCount int `json:"cloudConnectorCount"`
}
// Sends a GET request to the Disruptive REST API and returns the response body.
func getRequest(endpointURL string, queryParams map[string]string) ([]byte, error) {
// Create the HTTP GET request
req, err := http.NewRequest("GET", endpointURL, nil)
if err != nil {
return nil, err
}
// Set the query parameters for the request
q := req.URL.Query()
for key, value := range queryParams {
q.Set(key, value)
}
req.URL.RawQuery = q.Encode()
// Set the Authorization header to use HTTP Basic Auth
req.SetBasicAuth(keyID, secret)
// Send the HTTP request with a 3 second timout in case we
// are unable to reach the server.
client := &http.Client{Timeout: time.Second * 3}
response, err := client.Do(req)
if err != nil {
return nil, err
}
defer response.Body.Close()
return ioutil.ReadAll(response.Body)
}
// getAllProjects retrieves all projects that are accessible to the
// Service Account, one page at a time.
func paginatedGetProjects() ([]Project, error) {
// Initialize empty output slice.
projects := []Project{}
// Define the structure of each page.
type ProjectsPage struct {
Projects []Project `json:"projects"`
NextPageToken string `json:"nextPageToken"`
}
// Start with an empty page token to request the first page.
// We'll update this variable to point to the subsequent pages.
var pageToken = ""
// Loop until all pages have been fetched
for {
// Define the query parameters for the request.
// The "page_token" query parameter specifies the page
// we want to get. The "page_size" parameter is included
// just for the sake of completion. The default page size is 100.
params := map[string]string{
"page_token": pageToken,
"page_size": "100",
}
// Send GET request to the endpoint and get bytes back
pageBytes, err := getRequest(baseURL+"/projects", params)
if err != nil {
return nil, err
}
// Decode pageBytes into the expected page structure
page := ProjectsPage{}
if err := json.Unmarshal(pageBytes, &page); err != nil {
return nil, err
}
// Append the retrieved page of projects to our output slice
projects = append(projects, page.Projects...)
fmt.Printf("Got a page of %d projects\n", len(page.Projects))
// Update the page token to point to the next page
pageToken = page.NextPageToken
// If we got an empty "next_page_token" in the response,
// it means there are no more pages to fetch.
if len(pageToken) == 0 {
break
}
}
return projects, nil
}
func main() {
// Make paginated requests for all available projects.
projects, err := paginatedGetProjects()
if err != nil {
log.Fatal(err)
}
// Print display name of all fetched projects.
for i, project := range projects {
fmt.Printf("%d. %s\n", i, project.DisplayName)
}
}
The number of projects resulting from each page will be displayed in the stdout.
Paging...
- Got 100 projects in page.
- Got 100 projects in page.
- Got 100 projects in page.
- Got 68 projects in page.
0. my-project-A
1. my-project-B
2. my-project-C
3. ...
Last modified 8mo ago