How to write to a Local file(.csv) from Node.JS Proxy

[#1]

Hi,

I have a scenario: Convert the JSON into CSV and write the output to a local .CSV File.

For the above scenario I have created a proxy of type node.js and imported the 2 npm modules

1) jscon2csv

2)fs

my main .js file code is as below,

var json2csv = require('json2csv');
var fs = require('fs');
var express = require('express')
var app = express()
app.post('/convertJsonToCSV', function(req,res){
var fields = ['car', 'price', 'color'];
var myCars = [
  {
    "car": "Audi",
    "price": 40000,
    "color": "blue"
  }, {
    "car": "BMW",
    "price": 35000,
    "color": "black"
  }, {
    "car": "Porsche",
    "price": 60000,
    "color": "green"
  }
];
var csv = json2csv({ data: myCars, fields: fields });
fs.writeFile('D:\Ravi\file.csv', csv, function(err) {
  if (err) throw err;
  console.log(csv);
   res.send("done");
});
});
app.listen(3000);

I could see the converted csv output in the trace but not sure how to create a .CSV file in my local file system. Please suggest.

Regards

Ravi

[deboraelkin]

Please have a look at Understanding Edge support for Node.js modules, there are some restrictions on Node.js, in particular:

• Filesystem access is restricted to the directory where the Node.js script was launched: the /resources/node directory. Node.js scripts can read and write files within this directory, for instance as a temporary scratch area, but there are no guarantees as to how long the files will persist.

Cheers,

Deb

[#3]

Hi Deb,

Thanks for the reply.

I went through the link you have given above and you are right file system access is restricted.

If I want to write to a file in /resources/node directory also it is not working.

Since I can't have a file of type .CVS in node directory, I have created a file Script-1.js only and trying to write the content to it.

PFB my updated code.

 var json2csv = require('json2csv');
var fs = require('fs');
var express = require('express')
var apigee = require('apigee-access');
var app = express()
app.post('/convertJsonToCSV', function(req,res){
var fields = ['car', 'price', 'color'];
var myCars = [
  {
    "car": "Audi",
    "price": 40000,
    "color": "blue"
  }, {
    "car": "BMW",
    "price": 35000,
    "color": "black"
  }, {
    "car": "Porsche",
    "price": 60000,
    "color": "green"
  }
];
var csv = json2csv({ data: myCars, fields: fields });
apigee.setVariable('CSVData',csv);
fs.writeFile('Script-1.js', csv, function(err) {
  if (err) throw err;
  console.log(csv);
   res.send("done");
});
});
app.listen(3000);

I could see the converted CSV content in Trace but along with it I am getting error as below,

*** Starting script Error: request must be a valid HTTP request at /organization/environment/api/node_modules/apigee-access/lib/index.js:129 at /organization/environment/api/index.js:25 at handle (/organization/environment/api/node_modules/express/lib/router/layer.js:95) at next (/organization/environment/api/node_modules/express/lib/router/route.js:137) at dispatch (/organization/environment/api/node_modules/express/lib/router/route.js:112) at handle (/organization/environment/api/node_modules/express/lib/router/layer.js:95) at /organization/environment/api/node_modules/express/lib/router/index.js:281 at process_params (/organization/environment/api/node_modules/express/lib/router/index.js:335) at next (/organization/environment/api/node_modules/express/lib/router/index.js:275) at expressInit (/organization/environment/api/node_modules/express/lib/middleware/init.js:40) at handle (/organization/environment/api/node_modules/express/lib/router/layer.js:95) at trim_prefix (/organizat ion/environment/api/node_modules/express/lib/router/index.js:317) at /organization/environment/api/node_modules/express/lib/router/index.js:284 at process_params (/organization/environment/api/node_modules/express/lib/router/index.js:335) at next (/organization/environment/api/node_modules/express/lib/router/index.js:275) at query (/organization/environment/api/node_modules/express/lib/middleware/query.js:45) at handle (/organization/environment/api/node_modules/express/lib/router/layer.js:95) at trim_prefix (/organization/environment/api/node_modules/express/lib/router/index.js:317) at /organization/environment/api/node_modules/express/lib/router/index.js:284 at process_params (/organization/environment/api/node_modules/express/lib/router/index.js:335) at next (/organization/environment/api/node_modules/express/lib/router/index.js:275) at handle (/organization/environment/api/node_modules/express/lib/router/index.js:174) at handle (/organization/environment/api/nod e_modules/express/lib/application.js:174) at /organization/environment/api/node_modules/express/lib/express.js:39 at emit (events.js:98) at adaptorhttp.js:540 at domain.js:183 at domain.js:123 at adaptorhttp.js:539 at adaptorhttp.js:649 *** Starting script *** Starting script *** Starting script
enforcement	request
internal	false
ScriptName	index.js
type	ScriptExecutor

[deboraelkin]

Hi Ravi,

I think the error might be caused because there is a missing argument in the call to apigee.setVariable. You need to add the HTTPRequest object as the first argument. In your case it would be:

apigee.setVariable(req,'CSVData',csv); 

Try to strip down your code to the basics (ie: writing a file) and then build from there.

The following code works for me:

var http = require('http');


console.log('node.js application starting...');


var svr = http.createServer(function(req, resp) {
	var fs = require('fs');
	var apigee = require('apigee-access');
	var myStr = "Hi there!";
	fs.writeFile('temp.txt', myStr, function(err) {
	  if (err) throw err;
	  console.log("Wrote file");
	  data = fs.readFileSync('temp.txt');
	  console.log("Read from file");
	  myResponse = "These are the contents of file temp.txt: " + data;
          apigee.setVariable(req, 'myVar',myResponse);
	  console.log("Set variable myVar");
	  console.log("About to send response = " + myResponse);
          resp.end(myResponse);
	});
	
});


svr.listen(9000, function() {
	console.log('Node HTTP server is listening');
});

PS: I don't think json2csv is a supported package. You'll probably need to change the file format, or find another workaround (eg: send a request to a backend endpoint that can do the conversion and writing in the final destination directory for you. This is probably the cleanest way to do it)

Hope this helps

[#5]

Hi Deb,

Thank you for the reply,

I tried with your code and I could see the output in trace but again 'temp.txt' file is not getting created.

I refreshed the proxy, undeployed it and re-deployed still I don't get the file in any of the folders in edge.

[deboraelkin]

Hi Ravi, please note that the code is actually returning the content of the file in the response, so the file exists (if even for a little bit). You can try changing the fs.readFileSync to fs.readdirSync(".") to convince yourself.

As the documentation notes, there is no guarantee on how long the file will exist for, so I still think that your best option is to delegate the actual conversion and file writing to a backend endpoint via a service callout, if necessary.