Pagination

An example of how to use pagination to fetch a lot of data at once from the REST API.

Overview

If your application needs to fetch more data than a single request can handle, pagination can be used to seamlessly fetch blocks of the data, or pages, until completion. It splits a large request into smaller ones at the cost of a few extra lines of code. While this example shows how one can fetch projects with pagination, it can easily be expanded for other requests if desired.

Preliminaries

  • Basic Auth For simplicity, we here use Basic Auth for authentication. We recommend replacing this with an OAuth2 flow for integrations more complex than local experimentation.

  • Service Account Credentials You need to have created and know the credentials of a Service Account. Any role will suffice.

Projects API

A list of all available projects is accessible through the API under the following endpoint.

https://api.disruptive-technologies.com/v2/projects

For full details on the endpoint, see the REST API Reference.

Example Code

The provided example code can be summarized by the following points.

  • Projects are fetched in groups of 100 at a time.

  • The length of each page is printed.

  • When all projects have been fetched, print the total number of projects.

  • Iterate through and print the display name for each project.

Environment Setup

If you wish to run the code locally, make sure you have a working runtime environment.

Python 3.8
Python API (Beta)
Node.js 14
Go 1.16
Python 3.8

The following packages are required by the example code and must be installed.

pip install requests==2.25.1
Python API (Beta)

The latest version of our Python API can be installed through pip.

pip install --upgrade disruptive
Node.js 14

The following modules are required by the example code and must be installed.

npm install [email protected]
Go 1.16

No additional packages are required.

Add the following environment variables as they will be used to authenticate the API.

Bash
Bash
export DT_SERVICE_ACCOUNT_KEY_ID=<YOUR_SERVICE_ACCOUNT_KEY_ID> # [string]
export DT_SERVICE_ACCOUNT_SECRET=<YOUR_SERVICE_ACCOUNT_SECRET> # [string]
export DT_SERVICE_ACCOUNT_EMAIL=<YOUR_SERVICE_ACCOUNT_EMAIL> # [string]
export DT_PROJECT_ID=<YOUR_PROJECT_ID> # [string]

Source

The following code snippet implements pagination in a few languages.

Python 3.9
Python API (Beta)
Node.js 14
Go 1.16
Python 3.9
import os
import requests # pip install requests==2.25.1
# Environment variables for authentication credentials.
SERVICE_ACCOUNT_KEY_ID = os.environ.get('DT_SERVICE_ACCOUNT_KEY_ID')
SERVICE_ACCOUNT_SECRET = os.environ.get('DT_SERVICE_ACCOUNT_SECRET')
BASE_URL = 'https://api.d21s.com/v2'
def get_projects_auto_paginated():
# Initialize empty output list.
projects = []
# Create a parameters dictionary that contains an empty page token.
params: dict = {'pageToken': ''}
# Loop until all pages have been fetched.
print('Paging...')
while True:
# Send GET request for projects.
page = requests.get(
url=BASE_URL+'/projects',
params=params,
auth=(SERVICE_ACCOUNT_KEY_ID, SERVICE_ACCOUNT_SECRET),
).json()
# Concatenate page projects to output list.
projects += page['projects']
print('- Got {} projects in page.'.format(
len(page['projects'])
))
# Update parameters with next page token.
if len(page['nextPageToken']) > 0:
params['pageToken'] = page['nextPageToken']
else:
break
return projects
if __name__ == '__main__':
# Make paginated requests for all available projects.
projects = get_projects_auto_paginated()
# Print display name of all fetched projects.
for i, project in enumerate(projects):
print('{}. {}'.format(i, project['displayName']))
Python API (Beta)
import os
import disruptive as dt # pip install disruptive
# Environment variables for authentication credentials.
KEY_ID = os.environ.get('DT_SERVICE_ACCOUNT_KEY_ID')
SECRET = os.environ.get('DT_SERVICE_ACCOUNT_SECRET')
EMAIL = os.environ.get('DT_SERVICE_ACCOUNT_EMAIL')
if __name__ == '__main__':
# Authenticate using Service Account credentials.
dt.default_auth = dt.Auth.service_account(KEY_ID, SECRET, EMAIL)
# Make paginated requests for all available projects.
projects = dt.Project.list_projects()
# Print display name of all fetched projects.
for i, project in enumerate(projects):
print('{}. {}'.format(i, project.display_name))
Node.js 14
const axios = require('axios').default // npm install [email protected]
// Environment variables for authentication and target.
const serviceAccountKeyID = process.env.DT_SERVICE_ACCOUNT_KEY_ID
const serviceAccountSecret = process.env.DT_SERVICE_ACCOUNT_SECRET
baseUrl = 'https://api.d21s.com/v2'
async function getProjectsAutoPaginated() {
// Initialize empty output list.
let results = []
// Create a parameters dictionary that contains an empty page token.
let params = {'pageToken': ''}
// Loop until all pages have been fetched.
console.log('Paging...')
while (true) {
// Send GET request for projects.
projects = await axios({
method: 'GET',
url: baseUrl+'/projects',
auth: {
username: serviceAccountKeyID,
password: serviceAccountSecret,
},
params: params,
})
// Concatenate response contents to output list.
results.push(...projects.data['projects'])
let projectsInPage = projects.data['projects'].length
console.log(`- Got ${projectsInPage} projects in page.`)
// Update parameters with next page token.
if (projects.data.nextPageToken.length > 0) {
params.pageToken = projects.data.nextPageToken
} else {
break
}
}
return results
}
async function main () {
// Make paginated requests for all available projects.
let projects = await getProjectsAutoPaginated()
// Print display name of all fetched projects.
for (i = 0; i < projects.length; i++) {
console.log(`${i}. ${projects[i]['displayName']}`)
}
}
main().catch((err) => {console.log(err)});
Go 1.16
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"time"
)
const (
baseURL = "https://api.d21s.com/v2"
)
// Environment variables for authentication and target.
var keyID = os.Getenv("DT_SERVICE_ACCOUNT_KEY_ID")
var secret = os.Getenv("DT_SERVICE_ACCOUNT_SECRET")
// Struct that represents a project fetched from the API.
type Project struct {
Name string `json:"name"`
DisplayName string `json:"displayName"`
Inventory bool `json:"inventory"`
Organization string `json:"organization"`
OrganizationDisplayName string `json:"organizationDisplayName"`
Sensorcount int `json:"sensorCount"`
CloudConnectorCount int `json:"cloudConnectorCount"`
}
func getRequest(endpointURL string, pageToken string) ([]byte, error) {
// Create a custom http Client with timeout.
client := &http.Client{Timeout: time.Second * 3}
// Create the request object with method, URL, but no optional body.
req, err := http.NewRequest("GET", endpointURL, nil)
if err != nil {
return nil, err
}
// Add request parameters.
q := req.URL.Query()
q.Add("pageToken", pageToken)
req.URL.RawQuery = q.Encode()
// Set the request Authorization header to use HTTP Basic Authentication.
req.SetBasicAuth(keyID, secret)
// Send an HTTP request and return an HTTP response.
response, err := client.Do(req)
if err != nil {
return nil, err
}
defer response.Body.Close()
return ioutil.ReadAll(response.Body)
}
func getProjectsAutoPaginated() ([]Project, error) {
// Initialize empty output slice.
var projects []Project
// Define page structure.
type ProjectsPage struct {
Projects []Project `json:"projects"`
NextPageToken string `json:"nextPageToken"`
}
// Initialize empty page token.
var pageToken = ""
// Loop until all pages have been fetched.
fmt.Println("Paging...")
for {
// Send GET request to the endpoint and get bytes back.
pageBytes, err := getRequest(baseURL+"/projects", pageToken)
if err != nil {
return nil, err
}
// Decode pageBytes into an expected page structure.
var page ProjectsPage
if err := json.Unmarshal(pageBytes, &page); err != nil {
return nil, err
}
// Concatenate page projects slice to result slice.
projects = append(projects, page.Projects...)
fmt.Printf("- Got %d projects in page.\n", len(page.Projects))
// Update pageToken global variable.
if len(page.NextPageToken) > 0 {
pageToken = page.NextPageToken
} else {
break
}
}
return projects, nil
}
func main() {
// Make paginated requests for all available projects.
projects, err := getProjectsAutoPaginated()
if err != nil {
log.Fatal(err)
}
// Print display name of all fetched projects.
for i, project := range projects {
fmt.Printf("%d. %s\n", i, project.DisplayName)
}
}

Expected Output

The number of projects resulting from each page will be displayed in the stdout.

Paging...
- Got 100 projects in page.
- Got 100 projects in page.
- Got 100 projects in page.
- Got 100 projects in page.
- Got 68 projects in page.