Skip to content
Developerhome
Product homepage
X3

File uploading

  Less than to read

Before starting this step, please read the Data Ingestion Endpoint topic, especially about the File ingestion.

In order to upload a file to X3, you should choose the folder from the list of authorized folders while pairing between the demo app and the sandbox X3. Therefore, you have to implement a routine to retrieve authorized folders list. Then, when you upload a file to X3, you can track the progression as describe here.

First, you create the handlebars template for the file uploading import.handlebars in views folder:

  <h2>Data Ingestion in X3</h2>

  <div id="alertMessage" class="alert alert-warning alert-dismissible fade show " style="display: " role="alert">
      <span id="alertMessageContent"></span>
      <button type="button" class="close" onclick="$('#alertMessage').hide()">
          <span aria-hidden="true">&times;</span>
      </button>
  </div>
  <div id="successMessage" class="alert alert-success alert-dismissible fade show " style="display: " role="alert">
      <span id="successMessageContent"></span>
      <button type="button" class="close" onclick="$('#successMessage').hide()">
          <span aria-hidden="true">&times;</span>
      </button>
  </div>

  <form ref='uploadForm'
              id='uploadForm'
              action='/api/dataingestion'
              method='post'
              encType="multipart/form-data">
      <div class="input-group mb-3">
          <div class="input-group-prepend w-12-c">
              <span class="input-group-text w-100">Folder</span>
          </div>
              <select class="custom-select" id="folder" name="folder" onload="getFoldersList();">
                  <option selected>Choose...</option>
              </select>
      </div>

      <div class="form-group">
              <label for="exampleFormControlFile1">Choose a file to upload</label>
              <input type="file" class="form-control-file"  name="x3File" />
      </div>
      <input type="hidden" name="sessionId" id="sessionId" />
      <input type='submit' class="btn btn-primary mb-2" value='Upload' />
  </form>


  <script type="text/javascript" src="./js/foldersHelper.js"></script>
  <script type="text/javascript" src="./js/importHelper.js"></script>
  <script>
  openTracking('');
  getFoldersList();

  </script>

Then you add foldersHelper.js and importHelper.js in js folder to implement client side call to upload API on the same server.

  function getFoldersList(selectLastItem)
  {

      var request = new XMLHttpRequest();
      let sessionId = localStorage.getItem('sessionId');
      request.open('GET', "/api/folders?sessionId=" + sessionId, true);

      request.setRequestHeader("content-type","application/json");
      request.send();
      request.onreadystatechange = function() {

          if (request.readyState == 4 && (request.status == 200 || request.status == 0)) {
              var result = JSON.parse(request.responseText);

              if (result.folders) {
                  result.folders.forEach(folder => {
                      addFolder(folder.name, folder.$urls.dataSync, folder.$urls.dataService);
                  });
                  if (selectLastItem){
                      $('#folder option:last').prop('selected', true);
                  }
              }
              loadCurlCommands();

          } else if (request.readyState == 4 ) {
              var result = JSON.parse(request.responseText);
              $('#alertMessageContent').text(result.message);
              $('#alertMessage').show();
              loadCurlCommands();
          }
      };
  }

  function addFolder(folderName, syncUrl, graphQlUrl){
      let item = document.createElement("option");
      item.textContent = folderName;
      let syncUrlAttribute = document.createAttribute("syncUrl");
      syncUrlAttribute.value = syncUrl;
      item.setAttributeNode(syncUrlAttribute);
      let graphqlUrlAttribute = document.createAttribute("graphQLUrl");
      graphqlUrlAttribute.value = graphQlUrl;
      item.setAttributeNode(graphqlUrlAttribute);
      $("#folder").append(item);
  }

  function openTracking(trackingId)
  {
      if(trackingId) {
          window.open("/filetracking?id=" + trackingId,'_blank');
      }

  }

  $(document).ready(function () {
      let sessionId = localStorage.getItem('sessionId');
      $('input[name="sessionId"]').val(sessionId);
  });

For the file tracking, we add template filetracking.handlebars in views folder:

  <h2>File tracking</h2>

  <div id="alertMessage" class="alert alert-warning alert-dismissible fade show " style="display:none" role="alert">
      <span id="alertMessageContent"></span>
      <button type="button" class="close" onclick="$('#alertMessage').hide()">
          <span aria-hidden="true">&times;</span>
      </button>
  </div>
  <div id="successMessage" class="alert alert-success alert-dismissible fade show " style="display:none" role="alert">
      <span id="successMessageContent"></span>
      <button type="button" class="close" onclick="$('#successMessage').hide()">
          <span aria-hidden="true">&times;</span>
      </button>
  </div>

  <div class="input-group mb-3">
      <div class="input-group-prepend w-12-c">
          <span class="input-group-text w-100">Tracking ID</span>
      </div>
      <input type="text" class="form-control" id="file_tracking_id" value="">
  </div>

  <button type="button" class="btn btn-secondary" onclick="getTracking();">Track ID</button>

  <script type="text/javascript" src="./js/fileTrackingHelper.js"></script>

Then you add fileTrackingHelper.js in js folder to implement client side call to tracking API on the same server.

  function getTracking() {

      $('#alertMessage').hide();
      $('#successMessage').hide();

      var request = new XMLHttpRequest();
      request.open('POST', "/api/getTracking", true);
      let sessionId = localStorage.getItem('sessionId');
      var data = {
          "file_tracking_id" : document.getElementById('file_tracking_id').value,
          "sessionId": sessionId
      };

      request.setRequestHeader("content-type","application/json");
      request.send(JSON.stringify(data));
      request.onreadystatechange = function() {

          if (request.readyState == 4 && (request.status == 200 || request.status == 0)) {
              var result = JSON.parse(request.responseText);
              console.log(result);

              $('#successMessageContent').html("URL : " + result.$url
              + "<br /> creationDateTime : " + result.createdAt
              + "<br /> configurationCode : " + result.customerCode
              + "<br /> filename : " + result.fileName
              + "<br /> lastStatusDateTime : " + result.lastStatusUpdate
              + "<br /> status : " + result.status
              + "<br /> statusMessage : " + result.statusMessage
              + "<br /> folder : " + result.folder);

              $('#successMessage').show();
              loadCurlCommands();
          } else if (request.readyState == 4 ) {
              var result = JSON.parse(request.responseText);
              $('#alertMessageContent').text(result.message);
              $('#alertMessage').show();
              loadCurlCommands();
          }
      };
  }

Now, you will take care of API calls from client side implemented just before. There are :

  • /api/dataingestion: you create the dataIngestionHandler.js in lib folder
  let request = require('./requestLib');

  let ENDPOINT_URL = process.env.ENDPOINT_URL;

  exports.uploadFile = async function(req, res, tokenHandler) {
      await uploadFileToS3(req, res, tokenHandler);
  };

  let uploadFileToS3 = async function(req, res, tokenHandler) {
      if (Object.keys(req.files).length == 0) {
          res.redirect("/import?error=No file to upload");
          return;
      }

      if (!req.body.folder || req.body.folder.startsWith("Choose")) {
          res.redirect("/import?error=Please select a folder");
          return;
      }

      let file = req.files.x3File;

      try {
          let result = await getSignedUrl(req, req.body.folder, file.name, tokenHandler, true);

          await putToS3(req, result.uploadUrl, file);
          let splitTrackingUrl = result.trackingUrl.split('/');
          let trackingId = "";
          if (splitTrackingUrl.length > 0) {
              trackingId = splitTrackingUrl[splitTrackingUrl.length - 1];
          }

          res.redirect("/import?success=File upload succeed&trackingId=" + trackingId);
      } catch (e) {
          res.redirect("/import?error=" + e.message);
      }
  };


  let getSignedUrl = async function(req, folder, filename, tokenHandler, canRefreshToken) {

      console.log(ENDPOINT_URL + "/dataingestion/" + folder + "/?filename=" + filename);
      let result = await request.post(req, {headers: {'content-type' : 'application/json', 'Authorization': 'Bearer ' + tokenHandler.accessToken()}, url: ENDPOINT_URL + "/dataingestion/" + folder + "/?filename=" + filename}, true);
      if (result.statusCode === 201) {
          let body = JSON.parse(result.body);
          return body;
      } else if (result.statusCode === 401 && canRefreshToken) {
          await tokenHandler.getNewAccessTokenFromRefreshToken(req);
          return await getSignedUrl(folder, filename, tokenHandler, false);
      } else {
          let message = "An error occured";
          let body = JSON.parse(result.body);
          if (body && body.message) {
              message = body.message;
          }
          throw new Error(message);
      }
  };


  let putToS3 = async function(req, url, file) {
      let result = await request.put(req, {body: file.data.toString(), url: url}, true);
      if (result.statusCode !== 200) {
          let message = "An error occured";
          let body = JSON.parse(result.body);
          if (body && body.message) {
              message = body.message;
          }
        throw new Error(message);
      }
  };
  • /api/folders: you create the exportHandler.js in lib folder
  let request = require('./requestLib');

  exports.getFoldersList = async function (req, res, tokenHandler) {
      await getFolders(req, res, tokenHandler, true);
  };

  let getFolders = async function (req, res, tokenHandler, canRefreshToken) {

      console.log(process.env.ENDPOINT_URL + "/folders");
      try {
          let result = await request.get(req,{headers: {'content-type': 'application/json', 'Authorization': 'Bearer ' + tokenHandler.accessToken()}, url: process.env.ENDPOINT_URL + "/folders"}, true)
          if (result.statusCode === 200) {
              console.log(JSON.stringify(result));
              res.send(result.body);
          } else if (result.statusCode === 401 && canRefreshToken) {
              await tokenHandler.getNewAccessTokenFromRefreshToken(req);
              await getFolders(req, res, tokenHandler, false);
          } else {
              let message = "An error occured";
              let body = JSON.parse(result.body);
              if (body && body.message) {
                  message = body.message;
              }
              res.status(400).send('{"message": "' + message + '"}');
          }
      } catch (error) {
          res.status(400).send('{"message": "' + error + '"}');
      }

  };
  • /api/getTracking:you create the fileTrackingHandler.js in lib folder
  let request = require('./requestLib');

  exports.getTracking = async function(req, res, tokenHandler) {
      await getTracking(req, res, tokenHandler, true);
  };

  let getTracking = async function(req, res, tokenHandler, canRefreshToken) {
      try {
          let result = await request.get(req,{headers: {'content-type' : 'application/json', 'Authorization': 'Bearer ' + tokenHandler.accessToken()}, url: process.env.ENDPOINT_URL + "/dataingestion/tracking/" + req.body.file_tracking_id}, true);
          if (result && result.statusCode) {
              if (result.statusCode === 200) {
                  let body = JSON.parse(result.body);
                  res.send(body);
              } else if (result.statusCode === 401 && canRefreshToken) {
                  await tokenHandler.getNewAccessTokenFromRefreshToken(req);
              }  else {
                  let message = "An error occurred";
                  let body = JSON.parse(result.body);
                  if (body && body.message) {
                      message = body.message;
                  }
                  res.status(400).send('{"message": "' + message + '"}');
              }
          }
      } catch (error) {
          res.status(400).send('{"message": "' + error + '"}');
      }

  };

  let doRequest = function(context) {
      context.followRedirect = false;
      return new Promise(function (resolve, reject) {
          request.get(context, function (error, response, body) {
              if (!error) {
                  if (response.statusCode === 301) {
                      context.url = response.headers.location;
                      console.log("Redirect to " + context.url);
                      doRequest(context).then((response, error) => {
                          console.log(JSON.stringify(response));
                          if (error) {
                              reject(error);
                          } else {
                              resolve({statusCode: response.statusCode, body: response.body});
                          }

                      });
                  } else {
                      resolve({statusCode: response.statusCode, body: body});
                  }
              } else {
                  reject(error);
              }
          });
      });
  };

Then add new path of your application to main.js :

const express = require('express');
const exphbs = require("express-handlebars");
const fileUpload = require("express-fileupload");
const app = express();
const bodyParser = require('body-parser');
const fs = require('fs')
const https = require('https')

const config = require('./lib/config.js');
const callbackHandler = require('./lib/callbackHandler');
const tokenHandler = require('./lib/tokenHandler');
const requestLib = require('./lib/requestLib');
const exportHandler = require('./lib/exportHandler');
const dataInHandler = require('./lib/dataIngestionHandler');
const fileTrackingHandler = require('./lib/fileTrackingHandler');
const port = 5050;

let envColor = process.env.COLOR;
let bgColor;
let buttonColor;
switch (envColor) {
    case "red":
        bgColor = "danger";
        buttonColor = "warning";
        break;
    case "grey":
        bgColor = "dark";
        buttonColor = "secondary";
        break;
    default:
        bgColor = "primary";
        buttonColor = "info";
        break;
}

app.use('/', express.static('app'));
app.use(bodyParser.json());       // to support JSON-encoded bodies
app.use(fileUpload());


app.get('/config', function (req, res) {
    config.get(req, res);
});

app.get('/settings', function (req, res) {
    res.render("settings", {bgColor: bgColor, btColor: buttonColor});
});

let redirectUrl = process.env.REDIRECT_URL ? process.env.REDIRECT_URL : process.env.ENV_URL + "/callback";
let urlParams = process.env.ENDPOINT_URL + "/token/authorise?client_id=" + process.env.CLIENT_ID + "&state=astate&redirect_uri=" + redirectUrl;
app.get('/pairingRequest', function (req, res) {
    res.render("pairingRequest", {bgColor: bgColor, btColor: buttonColor, url: urlParams, config_url: process.env.CONFIG_URL});
});

app.get('/callback', function (req, res) {
    callbackHandler.handleCallback(req, res, bgColor, buttonColor);
});

app.get('/api/curlCommands', function (req, res) {
    requestLib.getCurlCommands(req, res);
});

app.post('/api/initiateAccessToken', async function (req, res) {
    tokenHandler.initiate(req, res);
});

app.post('/api/refreshAccessToken', async function (req, res) {
    tokenHandler.refresh(req, res);
});

app.post('/api/deleteRefreshToken', async function (req, res) {
    tokenHandler.delete(req, res);
});
app.get('/api/folders', async function (req, res) {
    exportHandler.getFoldersList(req, res, tokenHandler);
});

app.post('/api/dataingestion', async function (req, res) {
    dataInHandler.uploadFile(req, res, tokenHandler);
});

app.post('/api/getTracking', async function (req, res) {
    fileTrackingHandler.getTracking(req, res, tokenHandler);
});

app.post('/api/decodeToken', async function (req, res) {
    tokenHandler.decodeCurrentToken(req, res);
});

app.get('/generateAccessToken', function (req, res) {
    res.render("accessToken", {
        bgColor: bgColor,
        btColor: buttonColor,
        code: req.query.code,
        refresh: false,
        inputTitle: "Access code"
    });
})

app.get('/refreshAccessToken', function (req, res) {
    res.render("accessToken", {bgColor: bgColor, btColor: buttonColor, code: tokenHandler.refreshToken(), refresh: true, inputTitle: "Refresh token", allowDelete: true});
});
app.get('/import', function (req, res) {
    res.render("import", {
        bgColor: bgColor,
        btColor: buttonColor,
        error: req.query.error,
        displayError: req.query.error ? "block" : "none",
        success: req.query.success,
        displaySuccess: req.query.success ? "block" : "none",
        trackingId: req.query.trackingId
    });
});

app.get('/export', function (req, res) {
    res.render("export", {bgColor: bgColor, btColor: buttonColor, accessToken: tokenHandler.accessToken(), folder: process.env.FOLDER});
});

app.get('/filetracking', function (req, res) {
    res.render("filetracking", {bgColor: bgColor, btColor: buttonColor, accessToken: tokenHandler.accessToken(), trackingId: req.query.id});
});

app.engine('handlebars', exphbs({defaultLayout: 'main', layoutsDir: 'views/layouts'}));
app.set('view engine', 'handlebars');
app.set('views', 'views');

https.createServer({
    key: fs.readFileSync('server.key'),
    cert: fs.readFileSync('server.cert')
  }, app).listen(port, function () {
});

  • In the end of this step we have this file structure:

    Step 4

In VSCode, launch the debug to see the result at https://localhost:5050/import.

Step 4

Choose a file and a folder, then click on “Upload” to upload to X3, you should get this screenshot if everything is OK:

Step 4

When the file is uploaded successfully, a new tab with be opened with the tracking code return from API gateway:

Step 4

Congratulation, you uploaded a file to X3.

You can also download the VSCode project at the end of this step here. Please note, after downloading the zip file, you have to run npm i to install all package dependencies before debugging with VSCode.