File pooling and downloading
Less than to read
Before starting this step, please read the Data Ingestion Endpoint topic, especially about the Files listing and File delivery.
In order to download a file to X3, you should choose the folder from the list of authorized folders while pairing between the demo app and the sandbox X3. Therefore, you have to implement a routine to retrieve authorized folders list. Then, before downloading a file, you pool the available files list, then when a matching file becomes available, you download it.
First, you create the handlebars template for the file uploading export.handlebars in views folder:
<h2>Export files from X3</h2>
<div id="alertMessage" class="alert alert-warning alert-dismissible fade show " style="display:none" role="alert">
<span id="alertMessageContent"></span>
<button type="button" class="close" onclick="$('#alertMessage').hide()">
<span aria-hidden="true">×</span>
</button>
</div>
<div class="input-group mb-3">
<div class="input-group-prepend w-12-c">
<span class="input-group-text w-100">Access token</span>
</div>
<input type="text" class="form-control" id="access_token" value="">
</div>
<div class="input-group mb-3">
<div class="input-group-prepend w-12-c">
<span class="input-group-text w-100">Folder</span>
</div>
<select class="custom-select" id="folder" onload="getFoldersList();">
<option selected>Choose...</option>
</select>
</div>
<button type="button" class="btn btn-secondary" onclick="getFilesList();">Get files list</button>
<div class="list-group mt-3 mb-3">
</div>
<script type="text/javascript" src="./js/exportHelper.js"></script>
<script type="text/javascript" src="./js/foldersHelper.js"></script>
<script>
getFoldersList();
</script>
Then you add exportHelper.js in js folder to implement client side call to pool the files list API on the same server.
function getFilesList(link)
{
$(".list-group").empty();
$('#alertMessage').hide();
var request = new XMLHttpRequest();
request.open('POST', "/api/export", true);
let sessionId = localStorage.getItem('sessionId');
var data = {
"accessToken" : document.getElementById('access_token').value,
"folder": document.getElementById('folder').value,
"link": link,
"sessionId": sessionId
};
request.setRequestHeader("content-type","application/json");
request.send(JSON.stringify(data));
request.onreadystatechange = function() {
if (request.readyState == 4 && (request.status == 200 || request.status == 0)) {
var result = JSON.parse(request.responseText);
if (result.$previous) {
addNextPrevisous(result.$previous, "Previous");
}
if (result.items) {
result.items.forEach(item => {
if (item.key) {
addFile(item.key, item.downloadUrl, document.getElementById('access_token').value);
}
});
}
if (result.$next) {
addNextPrevisous(result.$next, "Next");
}
loadCurlCommands();
} else if (request.readyState == 4 ) {
var result = JSON.parse(request.responseText);
$('#alertMessageContent').text(result.message);
$('#alertMessage').show();
loadCurlCommands();
}
};
}
function addFile(key, url, accessToken){
let item = document.createElement("a");
item.setAttribute('href', '/api/download?filename=' + key + '&path=' + url + '&token=' + accessToken);
item.setAttribute('class', 'list-group-item list-group-item-action');
item.setAttribute('target', "_blank");
item.textContent = key;
$(".list-group").append(item);
}
function addNextPrevisous(link, text) {
let item = document.createElement("button");
item.setAttribute('type', 'button');
item.setAttribute('class', 'list-group-item list-group-item-secondary');
item.setAttribute('onclick', "getFilesList('"+ link + "');");
item.textContent = text;
$(".list-group").append(item);
}
Now, you will take care of API calls from client side implemented just before. There are /api/export and /api/download need to be intercepted, therefore we implement the server side interceptor: you create the exportHandler.js in lib folder:
let request = require('./requestHelper');
exports.getFilesList = async function (req, res, tokenHandler) {
await getFiles(req, res, tokenHandler, true);
};
exports.getFoldersList = async function (req, res, tokenHandler) {
await getFolders(req, res, tokenHandler, true);
};
exports.download = async function (req, res, tokenHandler) {
await downloadFile(req, res, tokenHandler, true);
};
let getFiles = async function (req, res, tokenHandler, canRefreshToken) {
try {
let path = "/";
if (req.body.link && req.body.link !== "") {
path += req.body.link;
} else {
path += req.body.folder;
}
let result = await request.get(req, {headers: {'content-type': 'application/json', 'Authorization': 'Bearer ' + tokenHandler.accessToken()}, url: process.env.ENDPOINT_URL + "/datadelivery" + path}, true);
if (result && result.statusCode) {
if (result.statusCode === 200) {
let body = JSON.parse(result.body);
res.send(body);
} else if (result.statusCode === 401 && canRefreshToken) {
await tokenHandler.getNewAccessTokenFromRefreshToken(req);
await getFiles(req, res, tokenHandler, false);
} else {
let message = "An error occured";
let body = JSON.parse(result.body);
if (body && body.message) {
message = body.message;
}
res.status(400).send('{"message": "' + message + '"}');
}
}
} catch (error) {
res.status(400).send('{"message": "' + error + '"}');
}
};
let downloadFile = async function (req, res, tokenHandler, canRefreshToken) {
try {
console.log(process.env.ENDPOINT_URL + "/datadelivery/" + req.query.path);
let result = await request.get(req,{
headers: {'content-type': 'application/json', 'Authorization': 'Bearer ' + tokenHandler.accessToken()},
url: process.env.ENDPOINT_URL + "/datadelivery/" + req.query.path
});
if (result && result.statusCode) {
if (result.statusCode === 200) {
res.setHeader('Content-type', "application/octet-stream");
res.setHeader('Content-disposition', 'attachment; filename=' + req.query.filename);
res.send(result.body);
} else if (result.statusCode === 401 && canRefreshToken) {
await tokenHandler.getNewAccessTokenFromRefreshToken(req);
await downloadFile(req, res, tokenHandler, false);
} else {
let message = "An error occured";
let body = JSON.parse(result.body);
if (body && body.message) {
message = body.message;
}
res.redirect('/callback?statusCode=' + response.statusCode + '&error=' + message);
}
}
} catch (error) {
res.status(400).send('{"message": "' + error + '"}');
}
};
let getFolders = async function (req, res, tokenHandler, canRefreshToken) {
console.log(process.env.ENDPOINT_URL + "/folders");
try {
let result = await request.get(req,{headers: {'content-type': 'application/json', 'Authorization': 'Bearer ' + tokenHandler.accessToken()}, url: process.env.ENDPOINT_URL + "/folders"}, true)
if (result.statusCode === 200) {
console.log(JSON.stringify(result));
res.send(result.body);
} else if (result.statusCode === 401 && canRefreshToken) {
await tokenHandler.getNewAccessTokenFromRefreshToken(req);
await getFolders(req, res, tokenHandler, false);
} else {
let message = "An error occured";
let body = JSON.parse(result.body);
if (body && body.message) {
message = body.message;
}
res.status(400).send('{"message": "' + message + '"}');
}
} catch (error) {
res.status(400).send('{"message": "' + error + '"}');
}
};
Then add new path of your application to main.js :
const express = require('express');
const exphbs = require("express-handlebars");
const fileUpload = require("express-fileupload");
const app = express();
const bodyParser = require('body-parser');
const fs = require('fs')
const https = require('https')
const config = require('./lib/config.js');
const callbackHandler = require('./lib/callbackHandler');
const tokenHandler = require('./lib/tokenHandler');
const requestLib = require('./lib/requestLib');
const exportHandler = require('./lib/exportHandler');
const dataInHandler = require('./lib/dataIngestionHandler');
const fileTrackingHandler = require('./lib/fileTrackingHandler');
const port = 5050;
let envColor = process.env.COLOR;
let bgColor;
let buttonColor;
switch (envColor) {
case "red":
bgColor = "danger";
buttonColor = "warning";
break;
case "grey":
bgColor = "dark";
buttonColor = "secondary";
break;
default:
bgColor = "primary";
buttonColor = "info";
break;
}
app.use('/', express.static('app'));
app.use(bodyParser.json()); // to support JSON-encoded bodies
app.use(fileUpload());
app.get('/config', function (req, res) {
config.get(req, res);
});
app.get('/settings', function (req, res) {
res.render("settings", {bgColor: bgColor, btColor: buttonColor});
});
let redirectUrl = process.env.REDIRECT_URL ? process.env.REDIRECT_URL : process.env.ENV_URL + "/callback";
let urlParams = process.env.ENDPOINT_URL + "/token/authorise?client_id=" + process.env.CLIENT_ID + "&state=astate&redirect_uri=" + redirectUrl;
app.get('/pairingRequest', function (req, res) {
res.render("pairingRequest", {bgColor: bgColor, btColor: buttonColor, url: urlParams, config_url: process.env.CONFIG_URL});
});
app.get('/callback', function (req, res) {
callbackHandler.handleCallback(req, res, bgColor, buttonColor);
});
app.get('/api/curlCommands', function (req, res) {
requestLib.getCurlCommands(req, res);
});
app.post('/api/initiateAccessToken', async function (req, res) {
tokenHandler.initiate(req, res);
});
app.post('/api/refreshAccessToken', async function (req, res) {
tokenHandler.refresh(req, res);
});
app.post('/api/deleteRefreshToken', async function (req, res) {
tokenHandler.delete(req, res);
});
app.get('/api/folders', async function (req, res) {
exportHandler.getFoldersList(req, res, tokenHandler);
});
app.post('/api/dataingestion', async function (req, res) {
dataInHandler.uploadFile(req, res, tokenHandler);
});
app.post('/api/getTracking', async function (req, res) {
fileTrackingHandler.getTracking(req, res, tokenHandler);
});
app.post('/api/decodeToken', async function (req, res) {
tokenHandler.decodeCurrentToken(req, res);
});
app.get('/api/download', function (req, res) {
exportHandler.download(req, res, tokenHandler);
});
app.post('/api/export', function (req, res) {
exportHandler.getFilesList(req, res, tokenHandler);
});
app.get('/generateAccessToken', function (req, res) {
res.render("accessToken", {
bgColor: bgColor,
btColor: buttonColor,
code: req.query.code,
refresh: false,
inputTitle: "Access code"
});
})
app.get('/refreshAccessToken', function (req, res) {
res.render("accessToken", {bgColor: bgColor, btColor: buttonColor, code: tokenHandler.refreshToken(), refresh: true, inputTitle: "Refresh token", allowDelete: true});
});
app.get('/import', function (req, res) {
res.render("import", {
bgColor: bgColor,
btColor: buttonColor,
error: req.query.error,
displayError: req.query.error ? "block" : "none",
success: req.query.success,
displaySuccess: req.query.success ? "block" : "none",
trackingId: req.query.trackingId
});
});
app.get('/export', function (req, res) {
res.render("export", {bgColor: bgColor, btColor: buttonColor, accessToken: tokenHandler.accessToken(), folder: process.env.FOLDER});
});
app.get('/filetracking', function (req, res) {
res.render("filetracking", {bgColor: bgColor, btColor: buttonColor, accessToken: tokenHandler.accessToken(), trackingId: req.query.id});
});
app.engine('handlebars', exphbs({defaultLayout: 'main', layoutsDir: 'views/layouts'}));
app.set('view engine', 'handlebars');
app.set('views', 'views');
https.createServer({
key: fs.readFileSync('server.key'),
cert: fs.readFileSync('server.cert')
}, app).listen(port, function () {
});
-
In the end of this step we have this file structure:
In VSCode, launch the debug to see the result at https://localhost:5050/export.
Choose a folder, then click on “Get files list” to retrieve available files ready for downloading, you should get this screenshot if everything is OK:
Then you can click on the filename to download. You can follow this guide to setup a file export from X3
Congratulation, you downloaded a file from X3.
You can also download the VSCode project at the end of this step here. Please note, after downloading the zip file, you have to run npm i to install all package dependencies before debugging with VSCode.