Pagination

REST API Reference DT Studio Status Page
Search…
Pagination
An example of how to use pagination for fetching a lot of data at once from the REST API.
Overview
If your application needs to fetch more data than a single request can handle, pagination can be used to seamlessly fetch blocks of the data, or pages, until completion. It splits a large request into smaller ones at the cost of a few extra lines of code. While this example shows how one can fetch projects with pagination, it can be adapted for other requests if desired.
Preliminaries
Basic Auth
For simplicity, we here use Basic Auth for authentication. We recommend replacing this with an OAuth2 flow for integrations more complex than local experimentation.
Service Account Credentials
You must create and know the credentials of a Service Account. Any role will suffice.
Projects API
A list of all available projects is accessible through the API under the following endpoint.
https://api.disruptive-technologies.com/v2/projects
For full details on the endpoint, see the REST API Reference.
Example Code
The following points summarize the provided example code.
Projects are fetched in groups of 100 at a time.
The length of each page is printed.
When all projects have been fetched, print the total number of projects.
Iterate through and print the display name for each project.
Environment Setup
If you wish to run the code locally, make sure you have a working runtime environment.
Python 3.8
Python API
Node.js 14
Go 1.16
The following packages are required by the example code and must be installed.
1
pip install requests==2.25.1
Copied!
The latest version of our Python API can be installed through pip.
1
pip install --upgrade disruptive
Copied!
The following modules are required by the example code and must be installed.
1
npm install [email protected]
Copied!
No additional packages are required.
Add the following environment variables as they will be used to authenticate the API.
Bash
1
export DT_SERVICE_ACCOUNT_KEY_ID=<YOUR_SERVICE_ACCOUNT_KEY_ID> # [string]
2
export DT_SERVICE_ACCOUNT_SECRET=<YOUR_SERVICE_ACCOUNT_SECRET> # [string]
3
export DT_SERVICE_ACCOUNT_EMAIL=<YOUR_SERVICE_ACCOUNT_EMAIL> # [string]
4
export DT_PROJECT_ID=<YOUR_PROJECT_ID> # [string]
Copied!
Source
The following code snippet implements pagination in a few languages.
Python 3.9
Python API
Node.js 14
Go 1.16
1
import os
2
import requests  # pip install requests==2.25.1
3
​
4
# Environment variables for authentication credentials.
5
SERVICE_ACCOUNT_KEY_ID = os.environ.get('DT_SERVICE_ACCOUNT_KEY_ID')
6
SERVICE_ACCOUNT_SECRET = os.environ.get('DT_SERVICE_ACCOUNT_SECRET')
7
​
8
BASE_URL = 'https://api.d21s.com/v2'
9
​
10
​
11
def get_projects_auto_paginated():
12
    # Initialize empty output list.
13
    projects = []
14
​
15
    # Create a parameters dictionary that contains an empty page token.
16
    params: dict = {'pageToken': ''}
17
​
18
    # Loop until all pages have been fetched.
19
    print('Paging...')
20
    while True:
21
        # Send GET request for projects.
22
        page = requests.get(
23
            url=BASE_URL+'/projects',
24
            params=params,
25
            auth=(SERVICE_ACCOUNT_KEY_ID, SERVICE_ACCOUNT_SECRET),
26
        ).json()
27
​
28
        # Concatenate page projects to output list.
29
        projects += page['projects']
30
        print('- Got {} projects in page.'.format(
31
            len(page['projects'])
32
        ))
33
​
34
        # Update parameters with next page token.
35
        if len(page['nextPageToken']) > 0:
36
            params['pageToken'] = page['nextPageToken']
37
        else:
38
            break
39
​
40
    return projects
41
​
42
​
43
if __name__ == '__main__':
44
    # Make paginated requests for all available projects.
45
    projects = get_projects_auto_paginated()
46
​
47
    # Print display name of all fetched projects.
48
    for i, project in enumerate(projects):
49
        print('{}. {}'.format(i, project['displayName']))
Copied!
1
import os
2
import disruptive as dt  # pip install disruptive
3
​
4
# Environment variables for authentication credentials.
5
KEY_ID = os.environ.get('DT_SERVICE_ACCOUNT_KEY_ID')
6
SECRET = os.environ.get('DT_SERVICE_ACCOUNT_SECRET')
7
EMAIL = os.environ.get('DT_SERVICE_ACCOUNT_EMAIL')
8
​
9
if __name__ == '__main__':
10
    # Authenticate using Service Account credentials.
11
    dt.default_auth = dt.Auth.service_account(KEY_ID, SECRET, EMAIL)
12
​
13
    # Make paginated requests for all available projects.
14
    projects = dt.Project.list_projects()
15
​
16
    # Print display name of all fetched projects.
17
    for i, project in enumerate(projects):
18
        print('{}. {}'.format(i, project.display_name))
Copied!
1
const axios = require('axios').default // npm install [email protected]
2
​
3
// Environment variables for authentication and target.
4
const serviceAccountKeyID  = process.env.DT_SERVICE_ACCOUNT_KEY_ID
5
const serviceAccountSecret = process.env.DT_SERVICE_ACCOUNT_SECRET
6
​
7
baseUrl = 'https://api.d21s.com/v2'
8
​
9
async function getProjectsAutoPaginated() {
10
    // Initialize empty output list.
11
    let results = []
12
​
13
    // Create a parameters dictionary that contains an empty page token.
14
    let params = {'pageToken': ''}
15
​
16
    // Loop until all pages have been fetched.
17
    console.log('Paging...')
18
    while (true) {
19
        // Send GET request for projects.
20
        projects = await axios({
21
            method: 'GET',
22
            url: baseUrl+'/projects',
23
            auth: {
24
                username: serviceAccountKeyID,
25
                password: serviceAccountSecret,
26
            },
27
            params: params,
28
        })
29
​
30
        // Concatenate response contents to output list.
31
        results.push(...projects.data['projects'])
32
        let projectsInPage = projects.data['projects'].length
33
        console.log(`- Got ${projectsInPage} projects in page.`)
34
​
35
        // Update parameters with next page token.
36
        if (projects.data.nextPageToken.length > 0) {
37
            params.pageToken = projects.data.nextPageToken
38
        } else {
39
            break
40
        }
41
    }
42
​
43
    return results
44
}
45
​
46
async function main () {
47
    // Make paginated requests for all available projects.
48
    let projects = await getProjectsAutoPaginated()
49
​
50
    // Print display name of all fetched projects.
51
    for (i = 0; i < projects.length; i++) {
52
        console.log(`${i}. ${projects[i]['displayName']}`)
53
    }
54
}
55
​
56
main().catch((err) => {console.log(err)});
Copied!
1
package main
2
​
3
import (
4
	"encoding/json"
5
	"fmt"
6
	"io/ioutil"
7
	"log"
8
	"net/http"
9
	"os"
10
	"time"
11
)
12
​
13
const (
14
	baseURL = "https://api.d21s.com/v2"
15
)
16
​
17
// Environment variables for authentication and target.
18
var keyID = os.Getenv("DT_SERVICE_ACCOUNT_KEY_ID")
19
var secret = os.Getenv("DT_SERVICE_ACCOUNT_SECRET")
20
​
21
// Struct that represents a project fetched from the API.
22
type Project struct {
23
	Name                    string `json:"name"`
24
	DisplayName             string `json:"displayName"`
25
	Inventory               bool   `json:"inventory"`
26
	Organization            string `json:"organization"`
27
	OrganizationDisplayName string `json:"organizationDisplayName"`
28
	Sensorcount             int    `json:"sensorCount"`
29
	CloudConnectorCount     int    `json:"cloudConnectorCount"`
30
}
31
​
32
func getRequest(endpointURL string, pageToken string) ([]byte, error) {
33
	// Create a custom http Client with timeout.
34
	client := &http.Client{Timeout: time.Second * 3}
35
​
36
	// Create the request object with method, URL, but no optional body.
37
	req, err := http.NewRequest("GET", endpointURL, nil)
38
	if err != nil {
39
		return nil, err
40
	}
41
​
42
	// Add request parameters.
43
	q := req.URL.Query()
44
	q.Add("pageToken", pageToken)
45
	req.URL.RawQuery = q.Encode()
46
​
47
	// Set the request Authorization header to use HTTP Basic Authentication.
48
	req.SetBasicAuth(keyID, secret)
49
​
50
	// Send an HTTP request and return an HTTP response.
51
	response, err := client.Do(req)
52
	if err != nil {
53
		return nil, err
54
	}
55
	defer response.Body.Close()
56
​
57
	return ioutil.ReadAll(response.Body)
58
}
59
​
60
func getProjectsAutoPaginated() ([]Project, error) {
61
	// Initialize empty output slice.
62
	var projects []Project
63
​
64
	// Define page structure.
65
	type ProjectsPage struct {
66
		Projects      []Project `json:"projects"`
67
		NextPageToken string    `json:"nextPageToken"`
68
	}
69
​
70
	// Initialize empty page token.
71
	var pageToken = ""
72
​
73
	// Loop until all pages have been fetched.
74
	fmt.Println("Paging...")
75
	for {
76
		// Send GET request to the endpoint and get bytes back.
77
		pageBytes, err := getRequest(baseURL+"/projects", pageToken)
78
		if err != nil {
79
			return nil, err
80
		}
81
​
82
		// Decode pageBytes into an expected page structure.
83
		var page ProjectsPage
84
		if err := json.Unmarshal(pageBytes, &page); err != nil {
85
			return nil, err
86
		}
87
​
88
		// Concatenate page projects slice to result slice.
89
		projects = append(projects, page.Projects...)
90
		fmt.Printf("- Got %d projects in page.\n", len(page.Projects))
91
​
92
		// Update pageToken global variable.
93
		if len(page.NextPageToken) > 0 {
94
			pageToken = page.NextPageToken
95
		} else {
96
			break
97
		}
98
	}
99
​
100
	return projects, nil
101
}
102
​
103
func main() {
104
	// Make paginated requests for all available projects.
105
	projects, err := getProjectsAutoPaginated()
106
	if err != nil {
107
		log.Fatal(err)
108
	}
109
​
110
	// Print display name of all fetched projects.
111
	for i, project := range projects {
112
		fmt.Printf("%d. %s\n", i, project.DisplayName)
113
	}
114
}
Copied!
Expected Output
The number of projects resulting from each page will be displayed in the stdout.
1
Paging...
2
- Got 100 projects in page.
3
- Got 100 projects in page.
4
- Got 100 projects in page.
5
- Got 100 projects in page.
6
- Got 68 projects in page.
Copied!
Examples
Streaming Events
Last modified 1mo ago
Copy link
Contents